diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ab33c7a..c77927f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,16 @@ jobs: # der es selbst gebaut wurde, und v2.11.4 bricht gegen ein Modul mit # `go 1.27.0` mit "the Go language version used to build # golangci-lint is lower than the targeted Go version" ab. - version: v2.13.2 + # + # Bump auf v2.14.0 (2026-10-11, PR #47): der Runner löst GO_VERSION + # "1.27" seit dem 10.10. auf go1.27.2 auf, und v2.13.2 (gebaut mit + # einem älteren 1.27) kann dessen Export-Daten nicht lesen — + # "export data version 5 is greater than maximum supported version + # 4", typecheck-Fehler auf dem ersten `import "fmt"`, Nightly auf + # root rot seit 38019738113 (10.10.) bei sonst grünen Jobs. v2.14.0 + # (24.09.) liest sie; lokal gegen go1.27.2 mit demselben + # .golangci.yml: 0 issues. + version: v2.14.0 working-directory: go args: --timeout=5m # Kein Analyse-Cache (2026-08-26): der Cache-Key der Action trägt das diff --git a/README.md b/README.md index 53cd8919..f61ad37c 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Running the server (Go daemon + PostgreSQL 18 + pgvector) is `docker compose up > **Upgrading from v4? Reach v4.38.0 first.** v5.0.0 removes the `CTX_{CHAT,CHAT_FALLBACK,EMBED,DREAM,DREAM_EMBED,RERANK}_*` connection variables: backends are configured in the pool (`ctx backends`) and nowhere else. **v4.38.0 is the last release that still reads those variables**, and one boot on it carries a `.env`-only configuration into the pool for you, then names every variable and leftover settings row you still have. Going straight from an older v4 to v5 is not fatal down to v4.19.0 — the env bridge is gone by then and you re-enter your topology by hand with `ctx backends seed` — but **below v4.19.0 it is a hard stop**: v5 folds migrations 001–113 into a single baseline and refuses a database that never applied 113. Migration path and rollback notes: [operations](docs/operations.md#migration-133-the-backend-tuple-rows-are-deleted--and-the-upgrade-hop-that-has-to-come-first). -**Models:** the tested reference stack is qwen3.6:27b for chat/synthesis/dream/digest, Qwen3-Embedding-8B (1024-dim, CPU-sized) for embeddings, and a bge-reranker-v2-m3 cross-encoder sidecar for rerank — but ctx is engine-agnostic: any OpenAI-compatible or Ollama API works, models are per-backend configuration. CPU-only setups work; a GPU shortens chat latency. Details: [Models & hardware](docs/operations.md#models--hardware). +**Models:** the tested reference stack is qwen3.6:27b for chat/synthesis/dream/digest, Qwen3-Embedding-8B (1024-dim, CPU-sized) for embeddings, and a bge-reranker-v2-m3 cross-encoder sidecar for rerank — but ctx is engine-agnostic: any OpenAI-compatible or Ollama API works, and the chat-class roles can run on the native Anthropic Messages API (`protocol: "anthropic"`); models are per-backend configuration. CPU-only setups work; a GPU shortens chat latency. Details: [Models & hardware](docs/operations.md#models--hardware). ## How LLMs use ctx diff --git a/docker-compose.yml b/docker-compose.yml index e0ab9aa7..6fd69c8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -362,8 +362,10 @@ services: # ── status.* (1) ── CTX_STATUS_CHANNEL_PROBE_INTERVAL: ${CTX_STATUS_CHANNEL_PROBE_INTERVAL:-} - # ── embed_backfill.* (4) ── + # ── embed_backfill.* (6) ── CTX_EMBED_BACKFILL_SYNC_CAP: ${CTX_EMBED_BACKFILL_SYNC_CAP:-} + CTX_EMBED_BACKFILL_SYNC_BUDGET: ${CTX_EMBED_BACKFILL_SYNC_BUDGET:-} + CTX_EMBED_BACKFILL_INTERVAL: ${CTX_EMBED_BACKFILL_INTERVAL:-} CTX_EMBED_BACKFILL_MAX_TOKENS: ${CTX_EMBED_BACKFILL_MAX_TOKENS:-} CTX_EMBED_BACKFILL_BACKOFF_BASE: ${CTX_EMBED_BACKFILL_BACKOFF_BASE:-} CTX_EMBED_BACKFILL_BACKOFF_CAP: ${CTX_EMBED_BACKFILL_BACKOFF_CAP:-} diff --git a/docs/api.md b/docs/api.md index 3bba05b6..234598b1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -441,13 +441,15 @@ message — the verdict lives in the body, like every `backend-test` outcome. **Validation guards** (create AND update, `422` with field errors): credential-carrier headers in `extra_headers` (`Authorization`, `Cookie`, `*-key`, `*-token`, …) and credential-semantic `extra_body` fields are rejected — provider keys go through `api_key_ref` (the *name* of a sealed secret, resolved in-memory only); a `base_url` carrying userinfo (`user:pass@host`) is rejected for the same reason — the URL is echoed unredacted by every list response, log line and the append-only audit (rows written before this guard load fine and are rejected on their next write); `locality` is cross-validated against `base_url` (a publicly routable host must be `external`); embed roles on external backends are blocked without `metadata.embed_equivalence_verified=true` (foreign quantization corrupts the shared vector space). `metadata.score_domain` (rerank dialect slot) must be one of `auto` | `logit` | `probability` — `auto` (the default for every row that does not set it) keeps the wire-container coupling: `results` ⇒ raw logits, `data` ⇒ calibrated [0,1] probability; explicit values override the coupling for dialect-mixing backends. Changing the *effective* score domain of a rerank-capable backend (create with a non-`auto` value included — the delete+create bypass is closed like the trust confirm's) requires `confirm_score_domain_change:true`; the 400 carries the full chain of change (effective immediately on all future rankings; the access-log score trail is only comparable up to the switch; no corpus reprocessing — no curation artifact persists rerank-derived data) and the confirmed write echoes it as a warning. Raising `trust` requires `confirm_trust_elevation:true`. Every mutation reloads the pool snapshot synchronously — `backend-update {"enabled":false}` is an instant brake, no restart; psql edits converge via the 053 NOTIFY trigger. -**OpenRouter (first external backend, G29).** `provider_class: "openrouter"` refines the openai wire: the request **always** carries `provider.zdr:true` + `provider.data_collection:"deny"`, independent of trust level — trust decides WHICH content may flow, the provider class decides whether the provider may store it. `extra_body.provider` entries merge but can only tighten (the force runs after the merge). The single escape is `metadata.allow_data_collection:true`, requiring `confirm_data_collection:true`. Responses feed telemetry: `usage.cost` → llmlog `cost_usd` (local NULL), the top-level `model` (that actually answered) overwrites the row's model, the response `id` lands in `metadata.provider_request_id`. A "no providers" rejection (zdr/deny filter leaves no provider) classifies as configuration-permanent (1h cooldown, no retry storm). `backend-test` additionally reports `credits_remaining`/`usage_usd` (`GET /v1/key`) and `zdr_endpoints` (`GET /v1/endpoints/zdr`). **`base_url` is the API root WITHOUT the version segment** — the wire paths append `/v1/...` themselves; a `base_url` ending in `/v1` double-segments to a 404, and since v4.25.4 create/update reject such a root with a 422 (all openai- and rerank-protocol backends, not just openrouter). +**OpenRouter (first external backend, G29).** `provider_class: "openrouter"` refines the openai wire: the request **always** carries `provider.zdr:true` + `provider.data_collection:"deny"`, independent of trust level — trust decides WHICH content may flow, the provider class decides whether the provider may store it. `extra_body.provider` entries merge but can only tighten (the force runs after the merge). The single escape is `metadata.allow_data_collection:true`, requiring `confirm_data_collection:true`. Responses feed telemetry: `usage.cost` → llmlog `cost_usd` (local NULL), the top-level `model` (that actually answered) overwrites the row's model, the response `id` lands in `metadata.provider_request_id`. A "no providers" rejection (zdr/deny filter leaves no provider) classifies as configuration-permanent (1h cooldown, no retry storm). `backend-test` additionally reports `credits_remaining`/`usage_usd` (`GET /v1/key`) and `zdr_endpoints` (`GET /v1/endpoints/zdr`). **`base_url` is the API root WITHOUT the version segment** — the wire paths append `/v1/...` themselves; a `base_url` ending in `/v1` double-segments to a 404, and since v4.25.4 create/update reject such a root with a 422 (all openai-, anthropic- and rerank-protocol backends, not just openrouter). + +**Anthropic (native Messages API, Migration 155).** `protocol: "anthropic"` speaks `POST {base_url}/v1/messages` directly (`base_url: "https://api.anthropic.com"`, the key from `api_key_ref` travels as `x-api-key`, `anthropic-version: 2023-06-01`) instead of Anthropic's OpenAI-compatible layer — which rejects every sampling parameter on the Claude 5 models (the first production query failed on `temperature`), and, as observed on the ku-heimdall install, drops `response_format` silently and offers neither prompt caching nor effort control. The request carries **no sampling parameters** (`temperature`/`top_p`/`top_k` are 400s on these models; `num_ctx` stays ctx's own prompt budget), the system prompt as a top-level block with `cache_control: ephemeral` (cache hits need a ≥512-token prefix; verify in `metadata.cache_read_input_tokens` of the `context_llm_log` row), `max_tokens` from the pipeline's `num_predict` (16000 when a caller sets none; **thinking tokens count against it** — the Claude 5 models think adaptively and cannot be told not to, so a model_map `think:false` is ignored on this wire; a role whose rows show `metadata.finish_reason = length` needs a larger `model_map` `params.max_tokens` for that role), and `output_config.effort`: the role's `model_map` `params.effort` beats the row's `extra_body.effort`, which beats the explicit default `medium` (the model-side default of Claude Opus 5.5); both carriers accept only `low|medium|high|xhigh|max` (422 otherwise); the field is always sent, so the row needs a model that takes it (the Claude 5 family; Sonnet 4.5 / Haiku 4.5 reject it, and on Opus 4.7/4.8 an omitted `thinking` field means no thinking — neither is a supported row). Every other `extra_body` key merges top-level (the hatch for `thinking.display`, `stop_sequences`, …; the row wins over model_map params on collision, as on the openai wire). **JSON mode is a no-op** on this wire: `output_config.format` (structured outputs) needs a JSON schema with `additionalProperties:false` on every object, which ctx's schemaless `json_object` mode does not have, so `ChatJSON` sends the same body as `Chat` and the pipelines' local parsers stay the validator — run Dream with `CTX_DREAM_JSON_MODE=off` (see [operations](operations.md#environment-variables)). Responses: text blocks are concatenated (thinking blocks skipped), `prompt_tokens` is the **total** prompt (`input_tokens` + `cache_read_input_tokens` + `cache_creation_input_tokens` — Anthropic's `input_tokens` is the uncached remainder only), the two cache counters land in `metadata`, the message id in `metadata.provider_request_id`, `cost_usd` stays NULL (ctx does not price Anthropic; read the console), and `stop_reason` is mapped onto the vocabulary the consumers read (`end_turn`/`stop_sequence`/`pause_turn`→`stop`, `max_tokens`/`model_context_window_exceeded`→`length`, `tool_use`→`tool_calls`, `refusal`→`content_filter`; unknown values pass through raw — a `refusal` after partial text still ends the turn with that text, marked `content_filter` in the `done` event). Errors classify like every other wire: 429 → rate-limit cooldown (`Retry-After` honoured), 529 `overloaded_error` → gateway (failover), 401 → auth, 400 → request-specific, 500 `api_error` → stop (the pool's HTTP-500 doctrine); the provider's error type prefixes the logged message, and the same types inside a mid-stream `error` event (HTTP 200) classify the same way. Row shape (422 on create and update): `api_key_ref` required, `locality: external`, `num_ctx > 0` (no window discovery on this wire — synthesis would fail closed), `provider_class: generic`, no `embed`/`dream-embed`/`rerank` roles (the Messages API has no such endpoints). `backend-test` probes `GET {base_url}/v1/models` with the key, then the 1-token chat. The web-chat stream path (`ChatStreamBackend`) maps the Messages API SSE events (`content_block_delta` text/`input_json_delta`, `message_delta` stop reason + usage, `error`) onto the same event contract as the openai wire, tools included (`tool_use`/`tool_result` blocks, consecutive tool results folded into one user turn). Three repairs keep a session alive on this wire: the answer's `thinking` blocks, when the model emits any, ride along in memory to the next iteration of the same turn and are replayed verbatim ahead of the `tool_use` they introduced (the reference asks that a turn whose `tool_use` still awaits its result keep its thinking block, and an altered block is rejected; live on 2026-10-11 a short tool turn carried no thinking block and the round trip was accepted — they are not persisted, later turns replay without them), a `tool_use` the history never answered (interrupted turn) gets a synthetic `is_error` `tool_result`, and tool calls that arrived without an id from an openai-wire row earlier in the session get synthetic ids. The system block's breakpoint covers the stable prefix only — the per-call `promptguard` nonce rule goes into a second, uncached block — and the stream path adds the top-level automatic `cache_control` so the growing history is read from cache across iterations and turns. Not available on this wire by construction: logprobs (the dream decide primitive falls back to its generating prompt and memoises the backend as decide-incapable), forced `tool_choice` (a 400 on Claude Opus 5.5; `auto` is what both wires send). ## Web chat sessions (F6, migration 056) The persistence layer, server-side tool harness and streaming HTTP endpoint for web chat. `context_chat_sessions` is **scope-owned**: list and delete key on the creating tenant's home scope, so a key never sees a foreign tenant's chats. It snapshots the creating key's `read_scopes` and carries a **monotone `max_sensitivity` high-water-mark**. Reading or continuing a session requires `session.read_scopes ⊆ caller.ReadScopes` (else 404, no oracle), closing the shadow-corpus channel against future least-privilege keys. The HWM rises with **every** appended message (in the same short transaction that assigns the message `seq`), so a credentials-touched session stays full-trust-only for its whole life. `context_chat_messages` records per-message `sensitivity` (fail-closed `credentials` default), tool-call metadata, telemetry and a gapless `seq` (`UNIQUE(session_id, seq)`). A turn claims its session via a short `busy_until` CAS — a second concurrent turn gets 409 without blocking, a crashed turn self-heals on expiry. Retention is off by default (`CTX_WEBCHAT_SESSION_RETENTION`). -The **harness** (`internal/chat`) drives the model loop (model call → tool execution → next call), re-resolving the F3 chat chain each iteration on `max(request, session HWM)` sensitivity (an empty chain ends the turn — never a silent escalation). Four read-only tools run under the session's `read_scopes` snapshot: **ctx_query** (hybrid retrieval, delegated to `/api/query` with `synthesize:false`+`include_content:true`, attributed to the real key), **ctx_search**, **ctx_get** (full block, paged past the window via a resumable `offset`), **ctx_recent**. Each result is annotated with `max(sensitivity)` of the blocks it carried; tools are offered only to a full-trust backend, and the closing call after the tool-budget cap carries no tools array. Tool errors return to the model as `{"error":…}` and never abort the turn. Events flow through a narrow `Sink` interface, so a future headless agent runner can drive the same loop without HTTP. +The **harness** (`internal/chat`) drives the model loop (model call → tool execution → next call), re-resolving the F3 chat chain each iteration on `max(request, session HWM)` sensitivity (an empty chain ends the turn — never a silent escalation). Each model call streams over the wire of the backend's `protocol` (`ChatStreamBackend`): the OpenAI-compatible SSE for `openai` and `ollama` rows, the native Messages API SSE for `anthropic` rows (tool calls as `tool_use`/`tool_result` blocks, see the Anthropic paragraph under [Backend pool](#backend-pool)) — the harness's event contract (`delta`, `tool_call_start`, `done`) is the same on both. Four read-only tools run under the session's `read_scopes` snapshot: **ctx_query** (hybrid retrieval, delegated to `/api/query` with `synthesize:false`+`include_content:true`, attributed to the real key), **ctx_search**, **ctx_get** (full block, paged past the window via a resumable `offset`), **ctx_recent**. Each result is annotated with `max(sensitivity)` of the blocks it carried; tools are offered only to a full-trust backend, and the closing call after the tool-budget cap carries no tools array. Tool errors return to the model as `{"error":…}` and never abort the turn. Events flow through a narrow `Sink` interface, so a future headless agent runner can drive the same loop without HTTP. **Endpoints** (auth required; `CTX_WEBCHAT_ENABLED=false` ⇒ 404): diff --git a/docs/architecture.md b/docs/architecture.md index 8eaf7242..0d90e714 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -**Stack:** Go 1.27 (matches `go.mod` and the `golang:1.27-alpine` build image), PostgreSQL 18 + pgvector 0.8.2, 84 SQL migrations (numbered 001–086; 066 is the deliberately deferred tenant-OAuth slot, 082 is the reserved W13 webhook slot). Dual-protocol inference (Ollama native or OpenAI-compatible) via any provider — configured per role in the [backend pool](#backend-pool-f3-migrations-053055) (`ctx backends`), one row per backend (see [operations](operations.md#backends)). +**Stack:** Go 1.27 (matches `go.mod` and the `golang:1.27-alpine` build image), PostgreSQL 18 + pgvector 0.8.2, 84 SQL migrations (numbered 001–086; 066 is the deliberately deferred tenant-OAuth slot, 082 is the reserved W13 webhook slot). Multi-protocol inference (Ollama native, OpenAI-compatible, or the native Anthropic Messages API) via any provider — configured per role in the [backend pool](#backend-pool-f3-migrations-053055) (`ctx backends`), one row per backend (see [operations](operations.md#backends)). ## Pipelines at a glance @@ -372,7 +372,7 @@ Since T10 the registry is **API-exposed** (no schema change) via the `type-*` ma ## Backend pool (F3, migrations 053–055) -`context_backends` replaces the hardwired primary+fallback pair with a declarative, role-routed, priority-ordered pool. Each row is one backend: `base_url`, wire `protocol` (`openai`/`ollama`/`rerank`), `provider_class` (`generic`/`llamacpp`/`openrouter`), a **trust level**, an egress `locality`, a `roles` list (`synthesis`, `translate`, `embed`, `rerank`, `dream`, `digest`, `chat`, `classify`, free-form), a per-role `model_map`, per-role `timeouts`, `priority` and `enabled`. Order/priority are pure DATA — no code path references backend names or priority constants. +`context_backends` replaces the hardwired primary+fallback pair with a declarative, role-routed, priority-ordered pool. Each row is one backend: `base_url`, wire `protocol` (`openai`/`ollama`/`anthropic`/`rerank` — `anthropic` is the native Messages API wire, chat-class roles only, see [api — Anthropic](api.md#backend-pool)), `provider_class` (`generic`/`llamacpp`/`openrouter`), a **trust level**, an egress `locality`, a `roles` list (`synthesis`, `translate`, `embed`, `rerank`, `dream`, `digest`, `chat`, `classify`, free-form), a per-role `model_map`, per-role `timeouts`, `priority` and `enabled`. Order/priority are pure DATA — no code path references backend names or priority constants. `context_disable_profiles` + `context_disable_profile_backends` (092) add named **disable profiles** — a profile takes its member backends out of every chain while active (maintenance / eject use-case). Membership is a real FK join (no name matching, rename/delete-safe by CASCADE), profiles carry a `scope` (`_global` for operator profiles) with `UNIQUE(scope, name)`, and the migration backfills the legacy gaming toggle as the reserved profile `eject` (label „Eject-Modus“) — value and member set copied losslessly from `gaming.active`/`gaming.disabled_backends`. The pool loads profiles + memberships into the same atomic snapshot as the backends (ORDER BY name — the status frame's diffKey needs a byte-stable order). The chain-time exclusion arm is live: an active profile takes its members out of every chain (`disabled by profile `, effective state `profile-disabled`, precedence disabled > profile-disabled > cooldown > active). Since U01-W5 the eject profile is the **sole** chain-time exclusion mechanism — the legacy `gaming.active`/`gaming.disabled_backends` settings keys and the parallel gaming arm were retired (the `gaming-mode`→`eject` alias shim stays as a compat surface, and the status payload's `gaming.active` field is derived live from the eject profile). @@ -386,7 +386,7 @@ The table is the **only** source of truth. Nothing seeds it at boot: until an op - **Digest freshness state (retired, migration 129)** — `context_digest_state` plus `trg_digest_dirty`/`mark_digest_dirty()` from the initial schema are gone. The table was verified dead (no Go consumer; the scheduler keeps its dirty flag in memory), but the trigger still fired on **every** block write and updated a SINGLETON row — one shared row lock per writer, i.e. a pure serialisation point at target scale with nothing on the other side of it. Two historical timestamps were dropped with it, deliberately and by explicit decision. Drop order is load-bearing (trigger → function → table): the function resolves its table by name at execution time, so a table dropped first would break not the migration but every subsequent write. - **Supersedes filtering** — temporal-gated removal of outdated blocks from query results. -- **Transport retry** — all inference HTTP calls (chat ollama/openai, embed, rerank) retry exactly once on transient transport failures (connection reset / EOF before any response bytes) via `internal/httpx`. Covers the keep-alive race with llama.cpp's cpp-httplib servers (~5s idle close vs Go connection reuse); HTTP status errors and context deadlines are never retried. Inference POSTs are stateless, so a replay is safe. +- **Transport retry** — all inference HTTP calls (chat ollama/openai/anthropic, embed, rerank) retry exactly once on transient transport failures (connection reset / EOF before any response bytes) via `internal/httpx`. Covers the keep-alive race with llama.cpp's cpp-httplib servers (~5s idle close vs Go connection reuse); HTTP status errors and context deadlines are never retried. Inference POSTs are stateless, so a replay is safe. - **Streaming tool-call wire** (`llm.ChatStream`) — streaming OpenAI-compatible chat with function calling: multi-turn message arrays, per-delta events, index-keyed tool-call assembly, arguments normalisation (llama.cpp JSON-string fragments and whole-object form yield identical calls), hardened against OpenRouter SSE comment frames and mid-stream error events inside HTTP-200 streams; usage falls back to llama.cpp `timings` incl. MTP draft-acceptance. Drives the web-chat harness (see [api](api.md#web-chat-sessions)). Since issue #26 the **non-streaming** chat path carries the provider stop reason too (`ChatResponse.FinishReason`, decoded from Ollama `done_reason` resp. OpenAI `choices[0].finish_reason`, raw and unnormalised, `""` when the provider reports none) — until then only the SSE path saw it, which left every background pipeline unable to tell a `length` truncation from malformed output. - **Decide primitive** (`llm.DecideChoice`, `llm.DecideOptions`) — a classification read off the FIRST generated token's top-logprobs instead of decoded text (System-One style). The caller phrases its question so the answer is one short label token (`A`…`E`, `Yes`/`No`); the OpenAI wire is asked for `logprobs`/`top_logprobs` (`Options.TopLogprobs`, JSON-tagged off so the Ollama options block never carries it), and `ChatResponse.TopLogprobs` hands the alternatives back as a distribution over the labels — renormalised probabilities, the raw label **mass** (share of belief that landed in the requested vocabulary at all) and a TypeSafe-style **confidence** `(n·peak−1)/(n−1)`. Rationale: on the serving hardware one output token costs ~100× an input token (decode 18–30 tok/s vs prefill ≈2.5k tok/s) and a decision needs none; there is also no JSON to truncate or drift. A backend that reports no logprobs (Ollama, some relays) or an answer outside the label set (`ErrNoLogprobs` / `ErrNoLabelMass`, `IsDecideFallback`) is a fallback signal, never a fabricated distribution. Measured 2026-09-20 against the goldbench gold sets: rerank nDCG@15 0.841 vs 0.648 for the generating judge (equal on the judge's parsed cases, no parse failures), dream links 0.605 vs 0.599, recurrence 0.885 = 0.885, at 0 output tokens. - **Embed cache** — content-hash-keyed embedding cache (`context_embed_cache`) to avoid re-embedding identical text across pipelines. diff --git a/docs/operations.md b/docs/operations.md index 487c31bc..69b5200b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -10,7 +10,7 @@ The wizard is the documented first-run path and is idempotent — it reports eve 1. **Config** — asks for base URL and API key, writes `~/.config/ctx/config` (mode 0600). 2. **Server** — reads `/health`. A `degraded` or `unhealthy` server is shown but accepted: an install whose [backend pool](#backends) is still empty answers `unhealthy` with HTTP 503 by design, and that is precisely the state the next step fixes. -3. **Backend pool** — lists the pool and, when no enabled `_global` row serves the `synthesis` or `embed` role, asks for host, protocol, chat model, embedding model and an optional API key, then runs the same seed `ctx backends seed` runs (same server-admin gate, same full-trust posture, same sealed-secret handling — the key is read without echo and never lands in the row). Afterwards, and whenever it finds a pool already populated, it **probes** each serving row and names the dead ones. A key below server-admin is not an error here: the step says so, points at `ctx backends seed`, and init carries on — hooks and statusline do not need an admin key. +3. **Backend pool** — lists the pool and, when no enabled `_global` row serves the `synthesis` or `embed` role, asks for host, protocol, chat model, embedding model and an optional API key (for the `anthropic` protocol: the chat model's context window, a separate embedding host and protocol, and the key), then runs the same seed `ctx backends seed` runs (same server-admin gate, same full-trust posture, same sealed-secret handling — the key is read without echo and never lands in the row). Afterwards, and whenever it finds a pool already populated, it **probes** each serving row and names the dead ones. A key below server-admin is not an error here: the step says so, points at `ctx backends seed`, and init carries on — hooks and statusline do not need an admin key. 4. **Version** — compares the local build against the latest GitHub release. 5. **Hooks** — `SubagentStart` / `SubagentStop` in Claude Code's `settings.json`. 6. **Statusline** — `ctx statusline` in the same file. @@ -114,7 +114,7 @@ The 29 names, to grep an inherited `.env` against — `CTX_CHAT_{HOST,API_KEY,PR | `CTX_DREAM_EVAL_CAP_RETRY_FACTOR` | `2` | hot | Scales the output cap of **one** retry after the dream link evaluation was truncated at that cap. Since the non-streaming chat path decodes the provider's stop reason, a cap hit is *detected* (`finish_reason` = `length`) rather than inferred; where a provider reports none, the completion-token heuristic still carries it, and `metadata.cap_hit_source` on the `context_llm_log` row names which of the two spoke. The retry re-sends the **identical** prompt — no shrinking, no candidate re-planning — so the cap is the single variable, and it writes its own log row (`metadata.cap_retry`), keeping token and cost accounting at one row per wire call. **Precedence:** the factor multiplies the *resolved* cap, i.e. after a `num_predict` / `max_tokens` param in the serving row's `model_map` (`applyModelParams`, walked in `llm/chain.go`) — a caller-side doubling would be a no-op on exactly those rows. **On a second cap hit** the block is booked as a **completed-but-inert eval** — `dream_eval_count` + 1, `dream_last_inert`, `CTX_DREAM_BACKOFF_INERT_OFFSET` on the curve — instead of the 5-minute transient cooldown. That is the point of the key: the transient path has no attempt counter, so before this a block whose prompt cannot fit re-burned one eval call every five minutes forever. The block is not abandoned; it re-dreams later on the back-off curve and links normally once `CTX_DREAM_NUM_PREDICT` or this factor is raised. **`1` or lower disables the retry** and restores the previous behaviour exactly (one call, parse error, transient cooldown, re-pick) — a true kill switch, nothing else changes with it off. **Limit:** on a serving row whose `extra_body` pins a numeric `max_tokens`, that value wins on the wire (`extra_body` is merged last, which is why `resolveMaxOut` ranks it above `Options`), so a scaled `num_predict` could not take effect. Such a row is detected and the retry **skipped** — plain parse error, one WARN, no inert booking; tune `extra_body.max_tokens` there instead. A negative value is fatal at boot / `422`s the settings write (**V19** — the off-semantics are the whole range ≤ 1, so `0` and `1` are ordinary settings and only a negative factor has no reading at all) | | `CTX_DREAM_DECIDE_TIE_ODDS` | `1.618034` (φ) | hot | Near-tie distance of **decide-mode** link writing, as an odds ratio: of one block's gated links, the most confident (the anchor) is written plus every link whose odds lie within this factor of the anchor's; the rest is not a tie and is dropped before the hard cap (`links_untied` on the `dream: decide links finished` log line). Relative, not a threshold, because the read probability is overconfident while the per-block ranking is reliable (calibration 2026-10-03, see architecture *Decide mode*). **`0` disables the rule** (every gated link up to the hard cap, the pre-rule behaviour); `1` keeps the anchor and exact ties only. A value in (0, 1), a negative one, NaN or +Inf is fatal at boot / `422`s the settings write (**V36**). Read only when `CTX_DREAM_DECIDE_MODE` is `eval` or `all` | | `CTX_DREAM_LANGUAGE` | empty | hot | The **corpus language**: one knob for every surface that produces readable text — the daily synthesis report (the 03:00 iteration and `POST /api/synthesize/daily`), the LLM **cluster labels** (see `_LABEL_*` below), the **root map's fixed scaffolding plus its number formatting** (headings, coverage wording, freeze clauses, footer; `1.190`/`97,9 %` vs `1,190`/`97.9%`), and the `digest.mode=stub` pointer that forwards the old topic map to it. The per-block dream pipeline is unaffected. Empty = legacy behavior: German report, title `Tagesbericht `, tag `tagesbericht`, German map scaffolding. A BCP-47-style tag switches title (`Daily Report `), tag (`daily-report`) and system prompt together; the **primary subtag** decides, so `de-DE` stays German. The map has exactly **two** scaffolding tables — empty or any `de*` tag keeps the frozen German, every other tag renders **English** — while the LLM-produced labels follow the tag itself, so an unusual tag yields English scaffolding around labels in that language rather than a German/foreign mix. ⚠ The title is half the `(category, title, scope)` upsert key — changing this **starts a new report series**, it does not rename the old one (see [api](api.md#endpoints)). Validated (V14): `^[a-z]{2,3}(-[a-z0-9]{2,8})*$`, ≤ 35 chars — a malformed value aborts boot / 422s the settings write, because the tag is interpolated into an LLM system prompt | -| `CTX_DREAM_JSON_MODE` | `strict` | hot | Wire policy of the four dream stages that **parse** their answer — link evaluation, keyword extraction, the recurrence confirm and the temporal Phase-2 review. `strict` is the historical wire and sends the dialect's JSON-mode marker (`response_format: {"type":"json_object"}` on the OpenAI path, top-level `"format":"json"` on the Ollama path); `off` sends plain chat, leaving the JSON contract in the prompt and the pipeline's **local parsers as the only validator** — which is what they already are on every well-formed answer, since nothing downstream trusts the grammar. Empty reads as `strict` (legacy sentinel, same doctrine as `CTX_DREAM_LANGUAGE`); any other value is fatal at boot / `422`s the settings write (**V20** — deliberately not a tolerant WARN-and-reset: an unknown value would silently mean `strict`, so a typo'd opt-out would keep the grammar on the wire while the settings surface renders the operator's value, and the symptom they were chasing is a slow backend, not a wrong one). **Why turn it off:** a grammar-enforcing runtime pays for the constraint in decode throughput (the report on issue #33 measures roughly a halving on an oMLX relay, with speculative decoding disabled while a grammar is active; not reproduced in this repo), and the dream pipeline is the throughput floor of the corpus. The key is hot, so it can be A/B-tested on a live install without a restart. **What it does not cover:** the **daily synthesis** never sends the marker under either setting — its answer is prose stored verbatim as the report body, so a grammar there is not a validator but a corruptor. **Risk when off:** answers that arrive as prose (rather than bare or fenced JSON) become parse errors on the stages listed as `FAIL` below — for evaluation and recurrence that is a transient cooldown and a re-pick, for temporal a silently dropped finding. Watch `metadata.parse_format` on the `context_llm_log` rows for a drift-form shift after flipping it | +| `CTX_DREAM_JSON_MODE` | `strict` | hot | Wire policy of the four dream stages that **parse** their answer — link evaluation, keyword extraction, the recurrence confirm and the temporal Phase-2 review. `strict` is the historical wire and sends the dialect's JSON-mode marker (`response_format: {"type":"json_object"}` on the OpenAI path, top-level `"format":"json"` on the Ollama path; **the `anthropic` path sends no marker under either setting** — structured outputs need a JSON schema ctx does not carry, so a dream backend on that wire behaves as `off` regardless, and setting `off` there only makes the row say what the wire does); `off` sends plain chat, leaving the JSON contract in the prompt and the pipeline's **local parsers as the only validator** — which is what they already are on every well-formed answer, since nothing downstream trusts the grammar. Empty reads as `strict` (legacy sentinel, same doctrine as `CTX_DREAM_LANGUAGE`); any other value is fatal at boot / `422`s the settings write (**V20** — deliberately not a tolerant WARN-and-reset: an unknown value would silently mean `strict`, so a typo'd opt-out would keep the grammar on the wire while the settings surface renders the operator's value, and the symptom they were chasing is a slow backend, not a wrong one). **Why turn it off:** a grammar-enforcing runtime pays for the constraint in decode throughput (the report on issue #33 measures roughly a halving on an oMLX relay, with speculative decoding disabled while a grammar is active; not reproduced in this repo), and the dream pipeline is the throughput floor of the corpus. The key is hot, so it can be A/B-tested on a live install without a restart. **What it does not cover:** the **daily synthesis** never sends the marker under either setting — its answer is prose stored verbatim as the report body, so a grammar there is not a validator but a corruptor. **Risk when off:** answers that arrive as prose (rather than bare or fenced JSON) become parse errors on the stages listed as `FAIL` below — for evaluation and recurrence that is a transient cooldown and a re-pick, for temporal a silently dropped finding. Watch `metadata.parse_format` on the `context_llm_log` rows for a drift-form shift after flipping it | | `CTX_DREAM_BACKOFF_MODE` / `_FACTOR` / `_MIN` / `_GRACE` / `_CAP` / `_INERT_OFFSET` | `exp` / `1.6` / `12h` / `0` / `45d` / `7` | hot | Re-dream back-off by eval count (`exp`/`log`/`linear`/`off`). Cooldown grows from `MIN` (n=0) to `CAP`: fresh blocks re-dream sub-day, mature blocks back off to the cap. `_MIN`/`_CAP` take a duration suffix — `h`/`d`/`w`/`m` (30d)/`y` (365d), e.g. `12h`, `45d`, `1w` (bare number = hours). `_INERT_OFFSET` starts a no-links cycle further up the curve | | `CTX_PROMPT_VERSION` | `v5.2` | hot | Generator-prompt version (`v5.2` default, `v6` opt-in graded confidence) | | `CTX_TIMEZONE` | `Europe/Berlin` | hot | Cyclic-temporal phase calculation | @@ -415,7 +415,7 @@ run journal is unchanged and still carries the class `block_write_failed` and no The backend pool (`context_backends`) is the living LLM configuration — `ctx backends` lists and manages it, the web UI at `/settings/backends` does the same visually. -Two ways in on a fresh installation. Interactively, [`ctx init`](#first-run-ctx-init) asks for host, protocol and the two models, seeds the pool and probes what it wrote — the same code path, wrapped in prompts. Non-interactively (scripts, CI, cloud-init), `ctx backends seed` writes the first two rows without any interaction: +Two ways in on a fresh installation. Interactively, [`ctx init`](#first-run-ctx-init) asks for host, protocol and the two models (for the `anthropic` protocol also the chat model's context window and a separate embedding host, since the Messages API serves no embeddings), seeds the pool and probes what it wrote — the same code path, wrapped in prompts. Non-interactively (scripts, CI, cloud-init), `ctx backends seed` writes the first two rows without any interaction: ```bash ctx backends seed --host http://gpu-host:11434 --model qwen3 --embed-model qwen3-embedding @@ -432,12 +432,22 @@ The SeedSpec file has one object per role; everything except `host` and `model` } ``` +Hosted Claude through the native Messages API (`protocol: "anthropic"`, see [api — Anthropic](api.md#backend-pool)) is a two-host spec: the chat leg needs the key (sealed into a secret, as below) and `num_ctx` (the wire has no window discovery), the embed leg stays on a local `ollama`/`openai` backend because the Messages API serves no embeddings. `base_url` is the API root without `/v1`: + +```json +{ + "chat": {"host": "https://api.anthropic.com", "protocol": "anthropic", "model": "claude-opus-5-5", "num_ctx": 100000, "api_key": "sk-ant-…"}, + "embed": {"host": "http://gpu-host:11434", "model": "qwen3-embedding"} +} +``` + What to know before running it: - **A server-admin key is required.** A tenant-admin's writes would be pinned to its own tenant scope — the seed refuses instead of reporting success over a shared pool that stays empty. - **The rows are seeded `full-trust`**, which is what lets them serve your own blocks (the pool's default block sensitivity is `credentials`, and a `public` backend is filtered out of that chain). Set `"trust"` per role in the spec to deviate deliberately. - **`api_key` never lands in the table.** It is sealed into an F2 secret first and the row carries only the reference, so the server needs a configured `CTX_SECRETS_KEY` (`openssl rand -hex 32`) before a spec with `api_key` can be seeded. Without one the run aborts and writes nothing — there is no plaintext fallback. - **It is idempotent per row.** A run interrupted between the two creates is completed by simply running it again. Rows *outside* the pair it manages abort the run instead (`--force` overrides only that check), so an already-configured pool is never touched by accident. +- **The anthropic leg is validated on the spec.** `protocol: "anthropic"` on the embed leg, without a key, or without `num_ctx` aborts before anything is sealed or written — the same rules the server enforces with a 422 on `backend-create`. - **A seed file is transient, not config.** It carries the api_key in the clear: `chmod 0600`, delete it after the seed, never commit it. - **A pool that exists is not a pool that serves.** `ctx init` probes each serving row and names the dead ones; `ctx backends test ` does it for a single row. Rows named `herbert-chat` / `llama-embed` on `http://localhost:11434` are the residue of an env bootstrap that ran with nothing configured — inside the ctx container that URL is the container itself. They have to *leave* the pool, and `--force` does not do that for you: it only skips the foreign-row guard and then creates the seed's own rows, leaving the legacy pair enabled at priority 100 next to them (dead members of every failover chain, one health-probe timeout each). Delete them first — `ctx backends delete ` — then run `ctx backends seed`. Parking them instead (`ctx backends update '{"enabled":false}'`) keeps them out of the chains but leaves the seed needing `--force`. @@ -447,7 +457,7 @@ To take a GPU host (e.g. `herbert`) down for maintenance without dropping in-fli ## Models & hardware -ctx is **engine-agnostic**: every LLM role speaks either the OpenAI-compatible or the Ollama wire protocol (per-backend `protocol`), so any provider serving one of those works — local llama.cpp / Ollama or a remote API. Models are configuration, not code: each `context_backends` row carries a per-role `model_map` (see [architecture — backend pool](architecture.md#backend-pool-f3-migrations-053055)). The models below are the **tested production reference**, not hard requirements. Beyond the LLM backends, ctx itself is a single Go daemon plus PostgreSQL 18 + pgvector — no GPU needs of its own. +ctx is **engine-agnostic**: every LLM role speaks the OpenAI-compatible or the Ollama wire protocol (per-backend `protocol`), and the chat-class roles (synthesis, translate, dream, digest, chat) can also run on the native Anthropic Messages API (`anthropic`, hosted Claude — no embeddings or rerank there), so any provider serving one of those works — local llama.cpp / Ollama or a remote API. Models are configuration, not code: each `context_backends` row carries a per-role `model_map` (see [architecture — backend pool](architecture.md#backend-pool-f3-migrations-053055)). The models below are the **tested production reference**, not hard requirements. Beyond the LLM backends, ctx itself is a single Go daemon plus PostgreSQL 18 + pgvector — no GPU needs of its own. | Role(s) | Reference model | Host type | Purpose | |---|---|---|---| @@ -477,6 +487,8 @@ The `llama-embed` sidecar (`docker-compose.override.yml`, Qwen3-Embedding-8B, `- **Backfill failure memo (migration 113, Achse 04 W04-2).** Both backfill paths — the scheduler arm (`backfillOneEmbedding`) and the query-path pre-search loop (`backfillPending`) — used to retry a NULL-embedding block **indefinitely** if it exceeded the embed window: no truncate, no skip, nothing written to the DB on failure, so the next pick landed on the exact same block (observed 2026-07-10: a 13k-char block at 4704 tokens against a 4096 window looped every ~20 s and blocked ~265 pending blocks behind it). This is closed structurally, not operationally: every failed embed attempt now upserts a row into `context_embed_failures` (`block_id`, `attempts`, `last_error` — normalized, ≤500 chars, never raw wire/embed-text content —, `last_class`, `next_attempt_at`), and both pick queries exclude a block whose `next_attempt_at` is still in the future. A non-oversize failure (backend down, transient wire error) gets an exponential backoff (`embed_backfill.backoff_base` × 2^(attempts-1), capped at `embed_backfill.backoff_cap`; defaults 1min/24h; the base must be > 0 — V21 rejects 0, which would mean "retry immediately" and a tight loop against a failing backend, issue #38) — the next PEEK reaches the next-oldest block instead of re-picking the same one. An oversize block — classified either by a conservative pre-wire token estimate (`len(text)/4` against `embed_backfill.max_tokens`, default 24000, chosen to sit under the live kv-unified pool's 32k minus interactive headroom — see the tuning section above) or by an HTTP 400 whose body names `exceed_context_size` — gets `next_attempt_at = 'infinity'` instead: permanently parked, never a blind 24h-in-slow-motion retry, but never silently dropped either (the memo row stays visible in `context_embed_failures`). Known limitation: `ClearEmbedding` does NOT currently clear the block's failure memo, so a content edit that shrinks a previously oversize-parked block below the token threshold does **not** reactivate it on its own — the `infinity` exclusion persists until the memo row is cleaned up (a janitor for this is not yet built; see the Achse-04 backlog). The query-path loop additionally carries a synchronous cap (`embed_backfill.sync_cap`, default 4): before this wave a large rest-transient (e.g. after a bulk ingest that outran the scheduler arm) made the *first* interactive query pay the entire nachzug inline; now it embeds at most `sync_cap` blocks per request and leaves the rest to the scheduler arm. Keeping blocks compact (~1–1.5k chars per the store guidance) or sizing the embed window above your largest block is still the actual fix for an oversize block's content — the memo mechanism only stops it from starving everything behind it. +**The dedicated backfill arm and the query-path time budget (tracking issue: force-push.me/damienmoon/ctx/issues/1).** Two gaps in the above showed up on a CPU embed backend (Qwen3-Embedding-8B at the documented ~23 tok/s) with a ~10k-token block. First, the scheduler arm ran only at the top of the dream loop, so with `dream.enabled=false` — the default — nothing backfilled pending embeddings in the background at all; the query path was the only path. `runEmbedBackfillLoop` is now its own goroutine, started regardless of Dream: a successful pick loops at once, an empty pick waits `embed_backfill.interval` (default 15 s), `0` turns the arm off (polled every 15 s for a hot re-enable). The dream loop keeps its inline call; the tx-wrapped `FOR UPDATE SKIP LOCKED` pick makes two pickers safe. Second, `sync_cap` bounds how *many* blocks a query embeds inline, not how *long*: one block that needs 7 minutes consumed the caller's whole 120 s deadline, the question's own embed was never admitted, and the query answered 500 instead of degrading. `embed_backfill.sync_budget` (seconds, default 30, `0` = unbounded) now bounds the whole pre-search loop per request; each wire call runs under what is left of it. A block that overruns the budget — or dies with the request context — is memoized with the class `caller_timeout`: the query path never picks it again (`EmbedFailureQueryPathExcludedPredicate`), the background arm picks it at once (`next_attempt_at = now()`), and a later wire failure there reclassifies it as `wire` with the ordinary backoff. The block stays FTS-visible meanwhile. Previously such a failure was memoized as `wire` and re-burned the deadline of every query that landed after the backoff lapsed. + **Backfill skips retrieval-excluded types (2026-07-25).** Both backfill paths now also skip a block whose *type* carries retrieval policy `excluded` in the block-type registry (shipped builtins: `checkpoint`, `system-meta`). Such a vector has no consumer at all — every ranked path filters against the registry's visible-type allowlist, so the embedding was written once and then read by nobody. The trigger was a live incident on 2026-07-25 with two independent arms: 33 Hermes "Compaction source" parts (`type=checkpoint`, ~36.5 kB each) saturated the CPU embed backend for hours, and the digest rebuild rewrites `topic-map-private` (`type=system-meta`, 73.7 kB ≈ ~32k real tokens) on **every** boot, which clears its embedding and bought ~60 min of CPU embed for a dead vector — recurring, forever. The filter is policy-driven, not a hardcoded type list (same mechanic as `test.sh` T19), and deliberately conservative in the fail-safe direction: only `_global`-`excluded` skips; `damped` and `aggregate-to-parent` stay embeddable, a block whose type has no registry row at all stays embeddable, and a tenant scope that overrides an excluded type to a visible policy re-opens that type for the backfill (the pick has no reader identity, so the reader-aware overlay rule is mirrored as the union over all possible readers). Two operational consequences: blocks of an excluded type keep `embedding IS NULL` forever — that is the intended end state, not a stuck queue — and the `/status` `embed_backlog` metric still counts them, so a corpus with such blocks shows a permanent backlog floor (live at the time of writing: 24 `checkpoint` + 1 `system-meta`). Existing `infinity` failure memos for such blocks become moot: the predicate no longer even sees the rows. Flipping a type's policy from `excluded` to a visible one makes its blocks embeddable again on the next cycle without any manual step. **Re-embed migration worker (Achse 04 W04-4).** With an active migration in `running` or `verifying`, a dedicated scheduler arm (`migrateOneEmbedding`, BACKGROUND class, defers to interactive demand) walks all blocks that carry an old-space vector (`embedding IS NOT NULL AND embedding_next IS NULL AND NOT is_archived`) oldest-first from a persistent cursor, embeds them via the model-map key `embed_next` on the regular embed chain, and writes `embedding_next`/`embed_model_next` — the serving pair stays byte-identical until the cutover. Operator knobs: `embed_migration.batch_per_cycle` (default 8, `<=0` disables the arm), `embed_migration.max_tokens` (default 24000, pre-wire oversize gate; `0` disables the estimate — the wire classification of HTTP 400 `exceed_context_size` stays), `embed_migration.backoff_base`/`backoff_cap` (defaults 1min/24h; the base must be > 0 — V21, same tight-loop reasoning as the backfill sibling) — all hot, global-only. Failure memos are migration-scoped rows in `context_embed_failures` (independent from the regular backfill's memos): transient wire errors back off exponentially, `oversize` and `sensitivity_ineligible` (no backend trusted for the block's sensitivity) park at `infinity` — visible in the memo table, surfaced in status/state.sh with W04-7. **If a migration sits in `paused` with `last_error = "model guard: resolved , expected "`**: the target backend's model_map lost (or never had) the `embed_next` key resolving to the migration's `to_model` — fix the backend row, then resume; the guard re-checks every cycle and exists because a missing key silently falls back to the OLD model (relabeling old vectors as migrated). The arm maintains a runtime partial index `idx_embedding_next_pending` (created `CONCURRENTLY` when a migration is active, INVALID leftovers from a crashed build are dropped and rebuilt automatically, dropped again once the migration is terminal) — seeing it in the catalog is normal only while a migration exists. Measured backend behavior (2026-07-24, CPU llama.cpp, kv-unified): a parallel embed that overflows the shared pool WAITS (no error path), and even below the pool limit a small interactive embed queues behind a large prefill — at ~20 tok/s CPU prefill a 24k-token block holds a slot for ~19 min, which is what `max_tokens` actually bounds. @@ -711,6 +723,10 @@ docker exec n8n-db-1 psql -U "$CONTEXT_DB_USER" -d "$CONTEXT_DB" -c \ WHERE metadata ? 'supersedes_repair';" ``` +### Migration 155: `anthropic` joins the backend wire protocols + +`context_backends.protocol` was pinned by a `CHECK` to `openai`, `ollama` and `rerank`. Migration 155 widens it to include `anthropic` (the native Messages API wire, [api — Anthropic](api.md#backend-pool)). It resolves the baseline's unnamed constraint from `pg_constraint` by table and definition, drops it and re-adds it as `context_backends_protocol_check`; no row changes, the column keeps its `DEFAULT 'openai'`. Nothing to do at upgrade time. **Downgrading** a daemon below this migration with an `anthropic` row still in the pool makes that row dispatch onto the wrong wire — the Ollama `/api/chat` path on the pipelines (the old fallthrough for unknown protocols), the OpenAI SSE path on web chat (the old stream path never read the protocol); the pool loads the row either way, validation runs only on create and update — so disable or delete such rows before rolling the binary back; the widened `CHECK` itself is harmless to an older binary. + ### `graph_overview.csr_loader`: the rebuild's input substrate `CTX_GRAPH_OVERVIEW_CSR_LOADER` (default `false`, hot) switches how the rebuild gets its graph into memory. It changes no result — the partition, the modularity and the intra-cluster degrees are byte-identical either way, and that identity is a gate, not a hope. diff --git a/go/internal/backends/backend.go b/go/internal/backends/backend.go index e9030b71..2aa44860 100644 --- a/go/internal/backends/backend.go +++ b/go/internal/backends/backend.go @@ -79,11 +79,35 @@ type Protocol string // Supported wire protocols. Empty is only valid where a field documents // inheritance (dream-embed); config validation V4 rejects everything else. +// +// ProtocolAnthropic is the native Anthropic Messages API (POST /v1/messages, +// x-api-key + anthropic-version headers). It is a chat-only wire: the +// Messages API serves no embeddings and no rerank, so validation keeps the +// embed/dream-embed/rerank roles off such a row. Unlike the openai wire it +// carries NO sampling parameters (the Claude 5 models answer 400 to +// temperature/top_p/top_k), caches the system prompt (cache_control +// ephemeral) and takes an effort level (output_config.effort) instead of a +// think toggle — see llm/anthropic.go. const ( - ProtocolOllama Protocol = "ollama" - ProtocolOpenAI Protocol = "openai" + ProtocolOllama Protocol = "ollama" + ProtocolOpenAI Protocol = "openai" + ProtocolAnthropic Protocol = "anthropic" ) +// AnthropicEfforts are the accepted output_config.effort values of the +// anthropic wire (claude-api reference, 2026-10). Validation rejects any +// other extra_body.effort / model_map params.effort at write time so a typo +// never reaches the provider as a 400 on every call. +var AnthropicEfforts = map[string]bool{ + "low": true, "medium": true, "high": true, "xhigh": true, "max": true, +} + +// AnthropicDefaultEffort is sent when neither the role's model_map params nor +// the row's extra_body name an effort. It matches the model-side default of +// Claude Opus 5.5 (medium) and is sent EXPLICITLY so the effective level is +// visible in every request body rather than implied by the model. +const AnthropicDefaultEffort = "medium" + // ThinkMode is the reasoning toggle as configured: "true" | "false" | "" // (= omit from the request). type ThinkMode string diff --git a/go/internal/backends/classify.go b/go/internal/backends/classify.go index 51dbbdee..995771e3 100644 --- a/go/internal/backends/classify.go +++ b/go/internal/backends/classify.go @@ -100,6 +100,12 @@ func (c ErrClass) Cooldown(retryAfter time.Duration) time.Duration { // Classify maps one attempt error onto the failover catalog. providerClass // refines provider-specific permanent failures (OpenRouter "no providers"). // Context errors must be checked FIRST: a canceled parent wraps everything. +// providerErrorTyper is implemented by llm.StreamError: the provider's error +// type of an error event delivered inside an HTTP-200 stream. +type providerErrorTyper interface { + ProviderErrorType() string +} + func Classify(err error, providerClass string) ErrClass { if err == nil { return ClassOK @@ -135,6 +141,30 @@ func Classify(err error, providerClass string) ErrClass { return ClassBadRequest // unknown 4xx: request-specific, no cooldown } } + // A mid-stream error event (HTTP 200, error object inside the SSE + // stream) carries the provider's error TYPE instead of a status code. + // The anthropic wire names the same types its HTTP errors do, so they map + // onto the classes the equivalent status would get — otherwise a 529 + // overloaded_error would fail over as HTTP but stop the chain as an + // event. Unknown types (OpenRouter's numeric codes) fall through to the + // stop-the-chain default below. + var pe providerErrorTyper + if errors.As(err, &pe) { + switch pe.ProviderErrorType() { + case "overloaded_error": + return ClassGateway + case "api_error": + return ClassServerFault + case "rate_limit_error": + return ClassRateLimit + case "billing_error": + return ClassBilling + case "authentication_error", "permission_error": + return ClassAuth + case "invalid_request_error", "not_found_error", "request_too_large": + return ClassBadRequest + } + } if httpx.IsBackendUnavailable(err) { return ClassTransport } diff --git a/go/internal/backends/protocol_anthropic_integration_test.go b/go/internal/backends/protocol_anthropic_integration_test.go new file mode 100644 index 00000000..d282e3da --- /dev/null +++ b/go/internal/backends/protocol_anthropic_integration_test.go @@ -0,0 +1,69 @@ +//go:build integration + +package backends_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + + "github.com/GottZ/ctx/internal/testdb" +) + +// TestMigration155ProtocolCheckAdmitsAnthropic runs the real migration chain +// on a prod-parity PostgreSQL and pins what Migration 155 changed: the +// (formerly unnamed) CHECK on context_backends.protocol now carries its +// explicit name, admits 'anthropic' next to the three original wires, and +// still rejects anything else with a check_violation (SQLSTATE 23514) — so a +// typo'd protocol cannot reach the pool and fall onto the Ollama path. +func TestMigration155ProtocolCheckAdmitsAnthropic(t *testing.T) { + pool := testdb.SetupTestDB(t) + ctx := context.Background() + + var def string + if err := pool.QueryRow(ctx, + `SELECT pg_get_constraintdef(oid) FROM pg_constraint + WHERE conrelid = 'context_backends'::regclass AND conname = 'context_backends_protocol_check'`, + ).Scan(&def); err != nil { + t.Fatalf("constraint context_backends_protocol_check not found after the chain: %v", err) + } + for _, want := range []string{"openai", "ollama", "rerank", "anthropic"} { + if !strings.Contains(def, "'"+want+"'") { + t.Errorf("constraint %q lacks %q", def, want) + } + } + + // Exactly one CHECK mentions the column — the DO block's catalog lookup + // stays unambiguous. + var n int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM pg_constraint + WHERE conrelid = 'context_backends'::regclass AND contype = 'c' + AND pg_get_constraintdef(oid) LIKE '%protocol%'`).Scan(&n); err != nil || n != 1 { + t.Errorf("CHECKs mentioning protocol = %d (err %v), want exactly 1", n, err) + } + + // The DROP/ADD of the constraint leaves the column's DEFAULT alone. + var def2 string + if err := pool.QueryRow(ctx, + `SELECT column_default FROM information_schema.columns + WHERE table_name = 'context_backends' AND column_name = 'protocol'`).Scan(&def2); err != nil || !strings.Contains(def2, "'openai'") { + t.Errorf("protocol column default = %q (err %v), want 'openai'", def2, err) + } + + if _, err := pool.Exec(ctx, + `INSERT INTO context_backends (name, base_url, protocol, api_key_ref, locality, num_ctx) + VALUES ('anthropic-it', 'https://api.anthropic.com', 'anthropic', 'anthropic-chat', 'external', 100000)`); err != nil { + t.Fatalf("anthropic row rejected by the widened CHECK: %v", err) + } + + _, err := pool.Exec(ctx, + `INSERT INTO context_backends (name, base_url, protocol) VALUES ('grpc-it', 'http://x', 'grpc')`) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "23514" || pgErr.ConstraintName != "context_backends_protocol_check" { + t.Fatalf("unknown protocol must fail the named CHECK with 23514, got %v", err) + } +} diff --git a/go/internal/backends/validate.go b/go/internal/backends/validate.go index b115f304..f8e1e964 100644 --- a/go/internal/backends/validate.go +++ b/go/internal/backends/validate.go @@ -118,6 +118,7 @@ func ValidateBackend(b *Backend) (warnings []string, errs []FieldError) { warnings = append(warnings, w...) errs = append(errs, e...) errs = append(errs, validateCarriers(b)...) + errs = append(errs, validateAnthropic(b)...) return warnings, errs } @@ -148,9 +149,9 @@ func validateIdentity(b *Backend) (errs []FieldError) { errs = append(errs, validateVersionedRoot(b)...) switch b.Protocol { - case ProtocolOpenAI, ProtocolOllama, "rerank": + case ProtocolOpenAI, ProtocolOllama, ProtocolAnthropic, "rerank": default: - errs = append(errs, FieldError{"protocol", "must be one of: openai, ollama, rerank"}) + errs = append(errs, FieldError{"protocol", "must be one of: openai, ollama, anthropic, rerank"}) } switch b.ProviderClass { @@ -221,10 +222,63 @@ func validateVersionedRoot(b *Backend) []FieldError { case b.Protocol == ProtocolOpenAI || b.Protocol == "rerank": return []FieldError{{"base_url", "base_url is the API root WITHOUT the version segment (the openai/rerank wire paths append /v1/... themselves) — a /v1 root doubles to /v1/v1/... and 404s chat, embeddings and rerank"}} + case b.Protocol == ProtocolAnthropic: + // Same doubling on the Messages API: the wire path is + // {base_url}/v1/messages, the probe {base_url}/v1/models. + return []FieldError{{"base_url", + "base_url is the API root WITHOUT the version segment (the anthropic wire appends /v1/messages itself) — use https://api.anthropic.com, not .../v1"}} } return nil } +// validateAnthropic covers the row shape the native Messages API wire needs +// (llm/anthropic.go). Every rule here is a runtime failure turned into a 422: +// a keyless row 401s on the first call, a non-external locality would drop +// api.anthropic.com from the egress audit, a missing num_ctx fails synthesis +// closed (promptguard.ErrUndeclaredWindow — there is no window discovery for +// this wire), a non-generic provider class would run the OpenRouter ZDR force +// or the llama.cpp /health probe against a provider that has neither, and an +// embed/rerank role would fall onto the Ollama /api/embed path (embed.go +// dispatches "anything but openai" there) with a vector-space corruption +// risk — the Messages API has no embeddings endpoint at all. +func validateAnthropic(b *Backend) (errs []FieldError) { + if b.Protocol != ProtocolAnthropic { + return nil + } + if strings.TrimSpace(b.APIKeyRef) == "" { + errs = append(errs, FieldError{"api_key_ref", "anthropic protocol requires api_key_ref (create the secret first; the key travels as x-api-key)"}) + } + if b.Locality != "" && b.Locality != LocalityExternal { + errs = append(errs, FieldError{"locality", "anthropic protocol requires locality 'external' (the Messages API is a hosted provider)"}) + } + if b.NumCtx <= 0 { + errs = append(errs, FieldError{"num_ctx", "anthropic protocol requires num_ctx > 0 (no window discovery on this wire; synthesis fails closed without a declared window)"}) + } + if b.ProviderClass != "" && b.ProviderClass != ProviderGeneric { + errs = append(errs, FieldError{"provider_class", "anthropic protocol requires provider_class 'generic'"}) + } + for _, r := range b.Roles { + if embedRoles[r] || r == RoleRerank { + errs = append(errs, FieldError{"roles", + fmt.Sprintf("role %q is not served by the anthropic protocol (the Messages API has no embeddings or rerank endpoint)", r)}) + } + } + if raw, ok := b.ExtraBody["effort"]; ok { + if v, isStr := raw.(string); !isStr || !AnthropicEfforts[v] { + errs = append(errs, FieldError{"extra_body", "effort must be one of: low, medium, high, xhigh, max"}) + } + } + for role, spec := range b.ModelMap { + if raw, ok := spec.Params["effort"]; ok { + if v, isStr := raw.(string); !isStr || !AnthropicEfforts[v] { + errs = append(errs, FieldError{"model_map", + fmt.Sprintf("role %q: params.effort must be one of: low, medium, high, xhigh, max", role)}) + } + } + } + return errs +} + // validateRoles covers role/model coverage, the embed+external hard block // and the dream-on-local warning. func validateRoles(b *Backend) (warnings []string, errs []FieldError) { diff --git a/go/internal/backends/validate_test.go b/go/internal/backends/validate_test.go index b4a28ffc..e7ccd22a 100644 --- a/go/internal/backends/validate_test.go +++ b/go/internal/backends/validate_test.go @@ -434,3 +434,83 @@ func TestScoreDomainResolution(t *testing.T) { t.Errorf("invalid value: ScoreDomain() = %q, want auto fallback", got) } } + +// anthropicBackend is the minimal row the native Messages API wire accepts: +// keyed, external, generic, with a declared window and a chat-class role. +func anthropicBackend() Backend { + b := validBackend() + b.Protocol = ProtocolAnthropic + b.Host = "https://api.anthropic.com" + b.APIKeyRef = "anthropic-chat" + b.NumCtx = 100000 + b.Roles = []string{RoleSynthesis, RoleTranslate, RoleDream, RoleDigest, RoleChat} + return b +} + +// TestValidateAnthropicHappyPath pins that the documented minimal row passes +// — every rule test below mutates exactly one field away from it, so a +// regression in the fixture itself would turn all of them green for the +// wrong reason. +func TestValidateAnthropicHappyPath(t *testing.T) { + b := anthropicBackend() + if _, errs := ValidateBackend(&b); len(errs) != 0 { + t.Fatalf("minimal anthropic row rejected: %v", errs) + } +} + +// TestValidateAnthropicRules walks the runtime failures the protocol turns +// into 422s: keyless (401 on first call), non-external locality (egress audit +// hole), missing num_ctx (synthesis fails closed, no window discovery), +// non-generic class (ZDR force / llama.cpp probe against the wrong provider), +// embed and rerank roles (no such endpoints — embed.go would silently take +// the Ollama path), and a mistyped effort in either carrier. +func TestValidateAnthropicRules(t *testing.T) { + cases := []struct { + name string + mut func(b *Backend) + field string + }{ + {"keyless", func(b *Backend) { b.APIKeyRef = "" }, "api_key_ref"}, + {"lan locality", func(b *Backend) { b.Host = "http://10.0.0.2:8080"; b.Locality = LocalityLAN }, "locality"}, + {"no num_ctx", func(b *Backend) { b.NumCtx = 0 }, "num_ctx"}, + {"openrouter class", func(b *Backend) { b.ProviderClass = ProviderOpenRouter }, "provider_class"}, + {"llamacpp class", func(b *Backend) { b.ProviderClass = ProviderLlamaCpp }, "provider_class"}, + {"embed role", func(b *Backend) { b.Roles = append(b.Roles, RoleEmbed) }, "roles"}, + {"dream-embed role", func(b *Backend) { b.Roles = append(b.Roles, RoleDreamEmbed) }, "roles"}, + {"rerank role", func(b *Backend) { b.Roles = append(b.Roles, RoleRerank) }, "roles"}, + {"extra_body effort typo", func(b *Backend) { b.ExtraBody = map[string]any{"effort": "mediumm"} }, "extra_body"}, + {"extra_body effort non-string", func(b *Backend) { b.ExtraBody = map[string]any{"effort": 2} }, "extra_body"}, + {"model_map effort typo", func(b *Backend) { + b.ModelMap = map[string]ModelSpec{"default": {Model: "m", Params: map[string]any{"effort": "ultra"}}} + }, "model_map"}, + {"versioned root", func(b *Backend) { b.Host = "https://api.anthropic.com/v1" }, "base_url"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b := anthropicBackend() + tc.mut(&b) + if _, errs := ValidateBackend(&b); !fieldHit(errs, tc.field) { + t.Errorf("want a %q error, got %v", tc.field, errs) + } + }) + } + + // Valid efforts pass in both carriers. + for _, e := range []string{"low", "medium", "high", "xhigh", "max"} { + b := anthropicBackend() + b.ExtraBody = map[string]any{"effort": e} + b.ModelMap = map[string]ModelSpec{"default": {Model: "m", Params: map[string]any{"effort": e}}} + if _, errs := ValidateBackend(&b); len(errs) != 0 { + t.Errorf("effort %q rejected: %v", e, errs) + } + } + + // The rules are protocol-bound: an openai row keeps its freedoms (keyless, + // no num_ctx, embed roles on a verified external backend are the existing + // contracts and must not regress). + b := validBackend() + b.NumCtx = 0 + if _, errs := ValidateBackend(&b); fieldHit(errs, "num_ctx") || fieldHit(errs, "api_key_ref") { + t.Errorf("anthropic rules leaked onto an openai row: %v", errs) + } +} diff --git a/go/internal/chat/chat_options_effort_test.go b/go/internal/chat/chat_options_effort_test.go new file mode 100644 index 00000000..a8e79279 --- /dev/null +++ b/go/internal/chat/chat_options_effort_test.go @@ -0,0 +1,34 @@ +package chat + +import ( + "testing" + "time" + + "github.com/GottZ/ctx/internal/backends" +) + +// TestChatOptionsCarriesModelMapEffort pins the review finding that the +// harness bypasses applyModelParams: a chat row's params.effort must still +// reach Options.Extra, the slot the anthropic stream wire reads — otherwise +// docs/api.md's "model_map params.effort beats extra_body" holds on the +// non-stream path only. +func TestChatOptionsCarriesModelMapEffort(t *testing.T) { + e := &Engine{cfg: Config{}.withDefaults(), now: time.Now} + b := backends.Backend{ModelMap: map[string]backends.ModelSpec{ + "chat": {Model: "claude-opus-5-5", Params: map[string]any{"effort": "xhigh", "temperature": 0.2}}, + }} + opts := e.chatOptions(b, 100) + if opts.Extra["effort"] != "xhigh" { + t.Errorf("Options.Extra = %v, want effort xhigh from model_map params", opts.Extra) + } + if opts.Temperature != 0.2 || opts.NumPredict != 100 { + t.Errorf("temperature/num_predict = %v/%d, want 0.2/100", opts.Temperature, opts.NumPredict) + } + + // Without the param the slot stays nil — the openai wire's extra_body + // merge and the anthropic default are untouched. + plain := e.chatOptions(backends.Backend{ModelMap: map[string]backends.ModelSpec{"chat": {Model: "m"}}}, 100) + if plain.Extra != nil { + t.Errorf("Options.Extra = %v, want nil without an effort param", plain.Extra) + } +} diff --git a/go/internal/chat/dispatch_invariant_test.go b/go/internal/chat/dispatch_invariant_test.go index fe7f3d19..f6424d36 100644 --- a/go/internal/chat/dispatch_invariant_test.go +++ b/go/internal/chat/dispatch_invariant_test.go @@ -125,6 +125,16 @@ func newWireServer(t *testing.T, wireHits *int64) *httptest.Server { fmt.Fprintf(w, `{"embeddings":[[%s]],"prompt_eval_count":7}`, vec) case "/v1/embeddings": // embed.go:185 (openai) fmt.Fprintf(w, `{"data":[{"embedding":[%s]}]}`, vec) + case "/v1/messages": // anthropic.go (non-stream) OR anthropic_stream.go + if bytes.Contains(body, []byte(`"stream":true`)) { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, "data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":10}}}\n\n"+ + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ok\"}}\n\n"+ + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":5}}\n\n"+ + "data: {\"type\":\"message_stop\"}\n\n") + return + } + fmt.Fprint(w, `{"id":"msg_1","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":10,"output_tokens":5}}`) case "/v1/rerank": // rerank.go:96 var req struct { Documents []string `json:"documents"` @@ -212,9 +222,24 @@ func TestDispatchAcquireEqualsWireAllSites(t *testing.T) { t.Fatalf("rerank: wired=%v err=%v", tel.Wired, err) } - // Six sites, each one physical wire contact under one lease. - if ca.count() != 6 || atomic.LoadInt64(&wireHits) != 6 { - t.Fatalf("after the six sites: acquires=%d wire=%d, want 6/6", ca.count(), atomic.LoadInt64(&wireHits)) + // --- Site 7: anthropic.go (native Messages API, non-stream) — reached + // through the same chain dispatcher as sites 1 and 2 (chatWithFormat). --- + if _, _, _, err := llm.ChatChain(ctx, []backends.Backend{invariantBackend(srv.URL, backends.ProtocolAnthropic)}, + "chat", "s", "u", llm.Options{}, "", time.Second, nil, llmAdm); err != nil { + t.Fatalf("anthropic non-stream: %v", err) + } + + // --- Site 8: anthropic_stream.go — the ChatStreamBackend dispatch under + // the same stream lease as site 3. --- + so = e.runStream(ctx, []backends.Backend{invariantBackend(srv.URL, backends.ProtocolAnthropic)}, + []llm.ChatMsg{{Role: "user", Content: "hi"}}, false, 0, 1, "sess", "key", backends.SensPublic, llmAdm, noopSink{}) + if !so.served || so.err != nil { + t.Fatalf("anthropic stream: served=%v err=%v", so.served, so.err) + } + + // Eight sites, each one physical wire contact under one lease. + if ca.count() != 8 || atomic.LoadInt64(&wireHits) != 8 { + t.Fatalf("after the eight sites: acquires=%d wire=%d, want 8/8", ca.count(), atomic.LoadInt64(&wireHits)) } // --- Failover walk (design W5: "inkl. Failover-Pfade"): a down primary @@ -231,8 +256,8 @@ func TestDispatchAcquireEqualsWireAllSites(t *testing.T) { "chat", "s", "u", llm.Options{}, "", time.Second, nil, llmAdm); err != nil || served == nil || len(attempts) != 2 { t.Fatalf("failover walk: served=%v attempts=%d err=%v", served, len(attempts), err) } - if ca.count() != 8 || atomic.LoadInt64(&wireHits) != 8 { - t.Fatalf("after failover: acquires=%d wire=%d, want 8/8", ca.count(), atomic.LoadInt64(&wireHits)) + if ca.count() != 10 || atomic.LoadInt64(&wireHits) != 10 { + t.Fatalf("after failover: acquires=%d wire=%d, want 10/10", ca.count(), atomic.LoadInt64(&wireHits)) } // --- Lease-free without wire contact (the cache-hit clause's structural @@ -261,14 +286,16 @@ func TestDispatchAcquireEqualsWireAllSites(t *testing.T) { // deliberately added/removed AND its acquire coupling is proven in // TestDispatchAcquireEqualsWireAllSites. var wirePin = map[string]int{ - "llm/client.go": 2, // /api/chat + /v1/chat/completions (non-stream) - "llm/stream.go": 1, // /v1/chat/completions (stream) - "embed/embed.go": 2, // /api/embed + /v1/embeddings - "rerank/rerank.go": 1, // /v1/rerank + "llm/client.go": 2, // /api/chat + /v1/chat/completions (non-stream) + "llm/stream.go": 1, // /v1/chat/completions (stream) + "llm/anthropic.go": 1, // /v1/messages (anthropic non-stream) + "llm/anthropic_stream.go": 1, // /v1/messages (anthropic stream) + "embed/embed.go": 2, // /api/embed + /v1/embeddings + "rerank/rerank.go": 1, // /v1/rerank } // TestWireSitePositiveListPinned pins the grep positive list: the ONLY -// http.NewRequestWithContext sites in internal/{llm,embed,rerank} are the six +// http.NewRequestWithContext sites in internal/{llm,embed,rerank} are the ones // listed above. A new un-listed wire file, or a changed count in a listed file, // turns this gate red — the static half of the I-D1 endpoint proof (§1.2: // admittiertheit itself is proven by the count gate, not by grep). diff --git a/go/internal/chat/engine.go b/go/internal/chat/engine.go index 745577a9..2609f86b 100644 --- a/go/internal/chat/engine.go +++ b/go/internal/chat/engine.go @@ -231,7 +231,10 @@ func (e *Engine) RunTurn(ctx context.Context, ar *auth.AuthResult, sess *store.C }); perr != nil { return fmt.Errorf("chat: persist assistant tool_calls: %w", perr) } - msgs = append(msgs, llm.ChatMsg{Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls}) + // ProviderBlocks ride along in memory for the next iteration of THIS + // turn (the anthropic wire needs its thinking block back ahead of the + // tool_use it introduced); they are not persisted. + msgs = append(msgs, llm.ChatMsg{Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls, ProviderBlocks: res.ProviderBlocks}) for _, call := range res.ToolCalls { if serr := sink.Event("tool_call", map[string]any{ @@ -426,7 +429,7 @@ func (e *Engine) runStream(ctx context.Context, chain []backends.Backend, msgs [ // fires after ChatStream drained the stream, so the lease spans // first byte to stream end. defer lease.Release() - res, err := llm.ChatStream(runCtx, b.Host, b.APIKey, model, msgs, tools, opts, b.ExtraBody, e.cfg.LLMTimeout, + res, err := llm.ChatStreamBackend(runCtx, b, model, msgs, tools, opts, e.cfg.LLMTimeout, func(ev llm.StreamEvent) error { started = true if ev.ContentDelta != "" { @@ -584,6 +587,11 @@ func (e *Engine) recordLLM(sessID, apiKeyID string, required backends.Sensitivit entry.PromptTokens = res.PromptTokens entry.CompletionTokens = res.CompletionTokens entry.Metadata["tool_calls"] = len(res.ToolCalls) + if res.ProviderRequestID != "" { + entry.Metadata["provider_request_id"] = res.ProviderRequestID + } + // anthropic wire: prompt-cache counters (no-op on the openai wire). + llm.ApplyCacheTelemetry(&entry, res.CacheReadTokens, res.CacheCreationTokens) } recordEntry(e.pool, entry) } @@ -602,6 +610,14 @@ func (e *Engine) tokenClamp(b backends.Backend, budgetLeft int) int { return v } +// chatOptions builds the stream call's Options from the chat role's +// model_map params. Deliberately NOT applyModelParams: the harness owns +// NumPredict (tokenClamp) and the think toggle (buildStreamBody's +// reasoning-off doctrine), so only the keys the stream wires read are +// honoured — temperature/top_p for the openai wire, effort for the anthropic +// wire (resolveAnthropicEffort reads Options.Extra["effort"], the same slot +// applyModelParams fills on the non-stream path, so a row's params.effort +// means the same thing on both paths). func (e *Engine) chatOptions(b backends.Backend, numPredict int) llm.Options { opts := llm.Options{Temperature: 0.7, NumPredict: numPredict} if spec := b.ModelFor("chat"); spec.Params != nil { @@ -611,6 +627,9 @@ func (e *Engine) chatOptions(b backends.Backend, numPredict int) llm.Options { if tp, ok := spec.Params["top_p"].(float64); ok { opts.TopP = tp } + if eff, ok := spec.Params["effort"].(string); ok && eff != "" { + opts.Extra = map[string]any{"effort": eff} + } } return opts } diff --git a/go/internal/cli/backends_seed.go b/go/internal/cli/backends_seed.go index 7677a29e..5dc3480d 100644 --- a/go/internal/cli/backends_seed.go +++ b/go/internal/cli/backends_seed.go @@ -281,8 +281,22 @@ func seedRowFor(in seedBackend, defaultName string, roles []string, label string } switch backends.Protocol(protocol) { case backends.ProtocolOllama, backends.ProtocolOpenAI: + case backends.ProtocolAnthropic: + // The Messages API is chat-only (no embeddings endpoint), keyed, and + // has no window discovery — the server's validation says the same + // (422), but a seed is the first contact of a fresh install and + // should fail on the spec, before any secret is sealed. + if label == "embed" { + return seedRow{}, fmt.Errorf("%s: protocol anthropic serves no embeddings — give the embed leg an ollama or openai backend", label) + } + if in.APIKey == "" && in.APIKeyRef == "" { + return seedRow{}, fmt.Errorf("%s: protocol anthropic requires api_key or api_key_ref", label) + } + if in.NumCtx <= 0 { + return seedRow{}, fmt.Errorf("%s: protocol anthropic requires num_ctx (the model's context window, e.g. 100000) — there is no window discovery on this wire", label) + } default: - return seedRow{}, fmt.Errorf("%s: protocol %q is not one of ollama, openai", label, protocol) + return seedRow{}, fmt.Errorf("%s: protocol %q is not one of ollama, openai, anthropic", label, protocol) } trust := strings.TrimSpace(in.Trust) if trust == "" { diff --git a/go/internal/cli/backends_seed_test.go b/go/internal/cli/backends_seed_test.go index 38cd8e0c..f7b033c5 100644 --- a/go/internal/cli/backends_seed_test.go +++ b/go/internal/cli/backends_seed_test.go @@ -146,6 +146,9 @@ func TestSeedRowsRejectBadSpecs(t *testing.T) { {"chat host missing", seedSpec{Chat: seedBackend{Model: "m"}, Embed: ok}, "chat: host is required"}, {"embed model missing", seedSpec{Chat: ok, Embed: seedBackend{Host: "http://x"}}, "embed: model is required"}, {"bad protocol", seedSpec{Chat: seedBackend{Host: "http://x", Model: "m", Protocol: "grpc"}, Embed: ok}, "protocol"}, + {"anthropic on embed", seedSpec{Chat: ok, Embed: seedBackend{Host: "https://api.anthropic.com", Model: "m", Protocol: "anthropic", APIKey: "k", NumCtx: 1}}, "no embeddings"}, + {"anthropic keyless", seedSpec{Chat: seedBackend{Host: "https://api.anthropic.com", Model: "m", Protocol: "anthropic", NumCtx: 100000}, Embed: ok}, "api_key"}, + {"anthropic no num_ctx", seedSpec{Chat: seedBackend{Host: "https://api.anthropic.com", Model: "m", Protocol: "anthropic", APIKey: "k"}, Embed: ok}, "num_ctx"}, {"bad trust", seedSpec{Chat: seedBackend{Host: "http://x", Model: "m", Trust: "sort-of"}, Embed: ok}, "trust"}, {"key and ref", seedSpec{Chat: seedBackend{Host: "http://x", Model: "m", APIKey: "a", APIKeyRef: "b"}, Embed: ok}, "mutually exclusive"}, {"negative num_ctx", seedSpec{Chat: seedBackend{Host: "http://x", Model: "m", NumCtx: -1}, Embed: ok}, "num_ctx"}, @@ -333,3 +336,23 @@ func TestSeedRowLeadRoleIsTheLegsReason(t *testing.T) { type errorString string func (e errorString) Error() string { return string(e) } + +// TestSeedRowsAnthropicChatLeg: the anthropic protocol seeds a keyed, +// windowed chat row next to a local embed row — the mixed-protocol pair a +// hosted-Claude install needs (the Messages API serves no embeddings). +func TestSeedRowsAnthropicChatLeg(t *testing.T) { + rows := mustRows(t, seedSpec{ + Chat: seedBackend{Host: "https://api.anthropic.com", Protocol: "anthropic", Model: "claude-opus-5-5", APIKey: "sk-ant", NumCtx: 100000}, + Embed: seedBackend{Host: "http://gpu:11434", Model: "qwen3-embed"}, + }) + chat := rows[seedChatName] + if chat.payload["protocol"] != "anthropic" || chat.payload["num_ctx"] != 100000 { + t.Errorf("chat payload = %v, want protocol anthropic with num_ctx 100000", chat.payload) + } + if chat.payload["api_key_ref"] != seedChatName+"-key" || chat.secretValue != "sk-ant" { + t.Errorf("chat key must seal into %s-key: payload=%v secret=%q", seedChatName, chat.payload, chat.secretValue) + } + if rows[seedEmbedName].payload["protocol"] != "ollama" { + t.Errorf("embed payload = %v, want the ollama default", rows[seedEmbedName].payload) + } +} diff --git a/go/internal/cli/init.go b/go/internal/cli/init.go index 3d9fdeea..cfe56229 100644 --- a/go/internal/cli/init.go +++ b/go/internal/cli/init.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "strconv" "strings" "time" @@ -392,24 +393,68 @@ func askSeedSpec(ask backendsAsk, missing []string) (seedSpec, error) { if host == "" { return seedSpec{}, errors.New("no host given — nothing was written") } - protocol := ask.line(" ? Protocol, ollama or openai [ollama]: ") + protocol := ask.line(" ? Protocol, ollama, openai or anthropic [ollama]: ") if protocol == "" { protocol = string(backends.ProtocolOllama) } chatModel := ask.line(" ? Chat model: ") embedModel := ask.line(" ? Embedding model: ") - key := "" + chat := seedBackend{Host: host, Protocol: protocol, Model: chatModel} + embed := seedBackend{Host: host, Protocol: protocol, Model: embedModel} + if backends.Protocol(protocol) == backends.ProtocolAnthropic { + // The Messages API is chat-only and keyed: the embed leg needs its + // own host and wire, the chat leg a declared window and the key. + numCtx, err := askNumCtx(ask, " ? Context window of the chat model in tokens [100000]: ", 100000) + if err != nil { + return seedSpec{}, err + } + chat.NumCtx = numCtx + embed.Host = ask.line(" ? Embedding host (e.g. http://localhost:11434): ") + if embed.Host == "" { + return seedSpec{}, errors.New("no embedding host given — the anthropic wire serves no embeddings, nothing was written") + } + embed.Protocol = ask.line(" ? Embedding protocol, ollama or openai [ollama]: ") + if embed.Protocol == "" { + embed.Protocol = string(backends.ProtocolOllama) + } + if answerIsYes(ask.line(" ? Does the embedding host need an API key? [y/N]: ")) { + v, err := ask.secret(" ? Embedding API key (not echoed): ") + if err != nil { + return seedSpec{}, err + } + embed.APIKey = v + } + v, err := ask.secret(" ? Anthropic API key (not echoed): ") + if err != nil { + return seedSpec{}, err + } + chat.APIKey = v + return seedSpec{Chat: chat, Embed: embed}, nil + } if answerIsYes(ask.line(" ? Does the host need an API key? [y/N]: ")) { v, err := ask.secret(" ? API key (not echoed): ") if err != nil { return seedSpec{}, err } - key = v + chat.APIKey, embed.APIKey = v, v + } + return seedSpec{Chat: chat, Embed: embed}, nil +} + +// askNumCtx reads a positive integer answer, the default for the empty +// line. Anything else is an error, not a silent fallback: a typo'd window +// ("100k") would otherwise seed a 100000-token prompt budget on a model +// whose window the operator meant to declare as ten times that. +func askNumCtx(ask backendsAsk, question string, def int) (int, error) { + answer := strings.TrimSpace(ask.line(question)) + if answer == "" { + return def, nil + } + n, err := strconv.Atoi(answer) + if err != nil || n <= 0 { + return 0, fmt.Errorf("context window %q is not a positive integer (tokens, e.g. 100000) — nothing was written", answer) } - return seedSpec{ - Chat: seedBackend{Host: host, Protocol: protocol, Model: chatModel, APIKey: key}, - Embed: seedBackend{Host: host, Protocol: protocol, Model: embedModel, APIKey: key}, - }, nil + return n, nil } func answerIsYes(answer string) bool { diff --git a/go/internal/cli/init_test.go b/go/internal/cli/init_test.go index ff8bb1bb..f2f85d47 100644 --- a/go/internal/cli/init_test.go +++ b/go/internal/cli/init_test.go @@ -733,3 +733,78 @@ func containsHelper(s, substr string) bool { } return false } + +// TestInitBackendsStepAnthropicAsksEmbedLegSeparately: choosing the anthropic +// wire turns the single-host wizard into a two-host one — the Messages API +// serves no embeddings — and collects the window and the key the chat leg +// needs without the "does the host need a key?" detour (it always does). +func TestInitBackendsStepAnthropicAsksEmbedLegSeparately(t *testing.T) { + m := newInitMock(t, serverAdmin, nil) + ask, secretCalls := scriptedAsk([]string{ + "https://api.anthropic.com", "anthropic", "claude-opus-5-5", "qwen3-embedding", + "", // context window → default 100000 + "http://gpu:11434", // embedding host + "", // embedding protocol → ollama default + "n", // embedding host needs no key + }, "sk-ant-live") + + var ok bool + out := captureStdout(t, func() { ok = runBackendsStep(m.client(), ask) }) + if !ok { + t.Fatalf("anthropic seed must succeed; output:\n%s", out) + } + if *secretCalls != 1 { + t.Errorf("the key prompt must run exactly once (%d calls)", *secretCalls) + } + if strings.Contains(out, "sk-ant-live") { + t.Error("the key was echoed") + } + byName := map[string]map[string]any{} + for _, c := range m.creates { + name, _ := c["name"].(string) + byName[name] = c + } + chat, embed := byName[seedChatName], byName[seedEmbedName] + if chat == nil || embed == nil { + t.Fatalf("wizard did not create both rows: %v", m.creates) + } + if chat["protocol"] != "anthropic" || chat["base_url"] != "https://api.anthropic.com" { + t.Errorf("chat row = %v", chat) + } + if n, _ := chat["num_ctx"].(float64); int(n) != 100000 { + // The mock round-trips the payload through JSON, hence float64. + if n2, ok := chat["num_ctx"].(int); !ok || n2 != 100000 { + t.Errorf("chat num_ctx = %v, want the 100000 default", chat["num_ctx"]) + } + } + if chat["api_key_ref"] != seedChatName+"-key" { + t.Errorf("chat api_key_ref = %v", chat["api_key_ref"]) + } + if embed["protocol"] != "ollama" || embed["base_url"] != "http://gpu:11434" { + t.Errorf("embed row = %v, want the separately asked ollama host", embed) + } + if _, has := embed["api_key_ref"]; has { + t.Errorf("the Anthropic key must not be attached to the embed row: %v", embed) + } +} + +// TestInitBackendsStepAnthropicRejectsBadWindow: a non-numeric context +// window aborts the step before anything is sealed or written — a silent +// default would seed the wrong prompt budget. +func TestInitBackendsStepAnthropicRejectsBadWindow(t *testing.T) { + m := newInitMock(t, serverAdmin, nil) + ask, secretCalls := scriptedAsk([]string{ + "https://api.anthropic.com", "anthropic", "claude-opus-5-5", "qwen3-embedding", "100k", + }, "sk-ant-live") + var ok bool + out := captureStdout(t, func() { ok = runBackendsStep(m.client(), ask) }) + if ok { + t.Fatalf("a bad window must fail the step; output:\n%s", out) + } + if !strings.Contains(out, "not a positive integer") { + t.Errorf("output lacks the reason:\n%s", out) + } + if *secretCalls != 0 || len(m.creates) != 0 { + t.Errorf("secret prompts %d, creates %d — want nothing asked or written", *secretCalls, len(m.creates)) + } +} diff --git a/go/internal/config/config.go b/go/internal/config/config.go index 55687c04..af0f5f96 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -1592,6 +1592,29 @@ type EmbedBackfillConfig struct { // unit test that never sets this field disables the cap rather than // silently blocking every backfillPending call. SyncCap int `key:"embed_backfill.sync_cap" env:"CTX_EMBED_BACKFILL_SYNC_CAP" default:"4" mut:"hot" tenancy:"global-only"` + // SyncBudget bounds Pfad A in SECONDS, next to SyncCap's count: the + // whole pre-search loop of one request runs inside this wall-clock + // budget, and each block's embed call runs under what is left of it. + // SyncCap alone could not stop one slow block from eating the caller's + // entire deadline (the tracking issue, force-push.me/damienmoon/ctx/issues/1: a ~10k-token block against a + // CPU embed backend at ~23 tok/s needs ~7 min, the CLI client's deadline + // is 120 s — the question's own embed was never admitted and the query + // answered 500 instead of degrading). A block that overruns the budget + // is memoized as caller_timeout (store.EmbedFailureCallerTimeout) and + // left to the background arm; the question embed keeps the rest of the + // deadline. Default 30 s (a quarter of the CLI deadline). 0 = unbounded, + // the pre-budget behavior (explicit opt-out, SyncCap convention). + SyncBudget time.Duration `key:"embed_backfill.sync_budget" env:"CTX_EMBED_BACKFILL_SYNC_BUDGET" default:"30" mut:"hot" tenancy:"global-only"` + // Interval is the idle wait in seconds of the dedicated background + // backfill arm (runEmbedBackfillLoop) between two empty picks. Until + // this key the only background backfill was the call at the top of the + // dream loop, so with dream.enabled=false (the default) NOTHING embedded + // pending blocks in the background and Pfad A was the only path (issue + // force-push.me/damienmoon/ctx/issues/1). The arm runs regardless of Dream; a successful + // pick loops at once, an empty pick waits this long. 0 = arm off + // (polled every 15 s for a hot re-enable, contract.recheck_interval + // convention). + Interval time.Duration `key:"embed_backfill.interval" env:"CTX_EMBED_BACKFILL_INTERVAL" default:"15" mut:"hot" tenancy:"global-only"` // MaxTokens is the pre-wire Oversize-Gate estimate threshold // (design/04 §4.4): len(embedText)/4 above this skips the block WITHOUT // a wire call (last_class='oversize', next_attempt_at='infinity' — diff --git a/go/internal/config/descriptions.go b/go/internal/config/descriptions.go index 3b653642..95ef754a 100644 --- a/go/internal/config/descriptions.go +++ b/go/internal/config/descriptions.go @@ -175,6 +175,8 @@ var keyDescriptions = map[string]string{ //nolint:gosec // G101-Fehlalarm: Schl "contract.recheck_interval": "Seconds between periodic schema-drift re-checks; 0 runs no checks while the ticker keeps polling so a later hot enable takes effect", "status.channel_probe_interval": "Seconds between latency probes of the four retrieval channels (semantic, fts_de, fts_en, trigram); 0 (default) disables the probe", "embed_backfill.sync_cap": "Maximum pending blocks a query embeds inline before its search runs; 0 = uncapped, the cap bounds first-query latency after a backlog", + "embed_backfill.sync_budget": "Seconds a query may spend embedding pending blocks before its own search; a block that overruns it is memoized as caller_timeout and left to the background arm, 0 = unbounded", + "embed_backfill.interval": "Seconds the background embed-backfill arm waits between two empty picks; it runs regardless of Dream, 0 = arm off (polled for a hot re-enable)", "embed_backfill.max_tokens": "Estimated-token threshold (length/4) above which a block is permanently parked as oversize without a wire call; 0 disables the pre-check", "embed_backfill.backoff_base": "Base delay in seconds of the exponential retry curve (base * 2^(attempts-1)) for transient embed failures. Must be > 0 (V21) — 0 would mean retry immediately, a tight loop against a failing embed backend", "embed_backfill.backoff_cap": "Upper bound in seconds on the embed-failure retry delay; a persistently failing block retries at most this often", diff --git a/go/internal/config/validate.go b/go/internal/config/validate.go index 2870b917..2cf804b8 100644 --- a/go/internal/config/validate.go +++ b/go/internal/config/validate.go @@ -790,7 +790,7 @@ func validateIntSigns(c *Config) []Issue { // - V4 (protocol typos: anything but ollama/openai fell silently onto the // ollama wire path → 404 against llama.cpp) is enforced at the same // boundary: validateIdentity rejects a row whose protocol is not one of -// openai, ollama, rerank. +// openai, ollama, anthropic, rerank. // - V1 (dual-runner VRAM WARN, β6), V12 (dream-embed credential boundary, // β5), V8 and V13 (rerank host reachability WARNs, β3) are NOT rebuildable // here and are not rebuilt: each compared fields of two tuples, and which diff --git a/go/internal/config/validate_test.go b/go/internal/config/validate_test.go index 44604181..d027bb74 100644 --- a/go/internal/config/validate_test.go +++ b/go/internal/config/validate_test.go @@ -393,7 +393,7 @@ func TestV17bIsAdvisoryOnly(t *testing.T) { // changes TYPE (seconds → int, or a struct field that stops being a // time.Duration) would leave the walk without anyone noticing. Raise this // number in the same commit that adds a duration key. -const wantDurationKeys = 45 +const wantDurationKeys = 47 // +2 on 2026-10-09: embed_backfill.sync_budget and embed_backfill.interval, both V17-covered // TestValidateRejectsNegativeOnEveryDurationKey is the registry-wide form of // V17 (issue #29): before it, exactly two of the seconds keys had a sign check diff --git a/go/internal/events/backfill_arm_integration_test.go b/go/internal/events/backfill_arm_integration_test.go new file mode 100644 index 00000000..55383e6c --- /dev/null +++ b/go/internal/events/backfill_arm_integration_test.go @@ -0,0 +1,124 @@ +//go:build integration + +// Rot-Gate for the tracking issue, force-push.me/damienmoon/ctx/issues/1, scheduler half: the embed-backfill +// arm (backfillOneEmbedding) was called ONLY at the top of runDreamLoop, so +// with dream.enabled=false — the registry default — no goroutine ever +// embedded a pending block in the background. Live: a block stayed pending +// for two hours while the scheduler logged guard and digest ticks only; the +// query path was the sole embedder, inside its caller's deadline. +// +// Rot is the premise, stated against the production wiring: a scheduler +// with DreamEnabled=false starts no dream worker, and nothing else in Run +// called backfillOneEmbedding before this wave. Grün is the dedicated arm +// (runEmbedBackfillLoop): started regardless of Dream, it embeds the pending +// block within a few intervals, and a caller_timeout memo (the query path's +// hand-off) does not hold it back. +// +// Run: `go test -tags=integration ./internal/events/ -run TestEmbedBackfillArm -count=1 -v`. +package events + +import ( + "context" + "testing" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/config" + "github.com/GottZ/ctx/internal/dispatch" + "github.com/GottZ/ctx/internal/store" + "github.com/GottZ/ctx/internal/testdb" +) + +func TestEmbedBackfillArm_RunsWithoutDream_Integration(t *testing.T) { + if testing.Short() { + t.Skip("integration test") + } + pool := testdb.SetupTestDB(t) + ctx := context.Background() + + srv := headOfLineEmbedServer(t) + bpool := backends.NewPool(nil, nil) + bpool.SeedSnapshotForTest([]backends.Backend{embedPoolRow("embed-a", srv.URL, 100)}) + d := dispatch.New(nil, dispatch.DefaultSettings()) // empty policy: pass-through + t.Cleanup(d.Close) + + cfg := &config.Config{EmbedBackfill: config.EmbedBackfillConfig{ + Interval: 100 * time.Millisecond, + MaxTokens: 0, + BackoffBase: time.Hour, + BackoffCap: 24 * time.Hour, + }} + // DreamEnabled=false: the RED premise — before this wave, Run started no + // goroutine that could reach backfillOneEmbedding in this configuration. + s := NewScheduler(pool, config.NewStore(cfg), bpool, StartupConfig{DreamEnabled: false}) + s.SetDispatcher(d) + + waitEmbedded := func(t *testing.T, within time.Duration) bool { + t.Helper() + deadline := time.Now().Add(within) + for time.Now().Before(deadline) { + if pendingCount(t, pool) == 0 { + return true + } + time.Sleep(25 * time.Millisecond) + } + return false + } + + t.Run("arm_embeds_a_pending_block_with_dream_off", func(t *testing.T) { + seedPendingBlock(t, pool, "arm-pending-normal", time.Hour) + defer clearBlocks(t, pool) + + armCtx, stop := context.WithCancel(ctx) + defer stop() + go s.runEmbedBackfillLoop(armCtx) + + if !waitEmbedded(t, 5*time.Second) { + t.Fatalf("block still pending after 5 s: the dedicated arm did not embed it (pending=%d)", pendingCount(t, pool)) + } + }) + + t.Run("arm_picks_a_caller_timeout_block_at_once", func(t *testing.T) { + seedPendingBlock(t, pool, "arm-handed-off", time.Hour) + defer clearBlocks(t, pool) + + // The query path's hand-off: a caller_timeout memo, due now. + var blockID string + if err := pool.QueryRow(ctx, `SELECT id FROM context_blocks WHERE title = 'arm-handed-off'`).Scan(&blockID); err != nil { + t.Fatalf("read block id: %v", err) + } + if err := store.RecordEmbedFailure(ctx, pool, blockID, store.EmbedFailureCallerTimeout, + store.NormalizeEmbedError(store.EmbedFailureCallerTimeout, "context deadline exceeded"), + cfg.EmbedBackfill.BackoffBase, cfg.EmbedBackfill.BackoffCap); err != nil { + t.Fatalf("record caller_timeout memo: %v", err) + } + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM context_embed_failures WHERE block_id = $1`, blockID) }) + + armCtx, stop := context.WithCancel(ctx) + defer stop() + go s.runEmbedBackfillLoop(armCtx) + + if !waitEmbedded(t, 5*time.Second) { + t.Fatalf("caller_timeout block still pending after 5 s: the arm must pick it at once, not after a backoff") + } + }) + + t.Run("interval_zero_turns_the_arm_off", func(t *testing.T) { + seedPendingBlock(t, pool, "arm-off-pending", time.Hour) + defer clearBlocks(t, pool) + + off := *cfg + off.EmbedBackfill.Interval = 0 + sOff := NewScheduler(pool, config.NewStore(&off), bpool, StartupConfig{DreamEnabled: false}) + sOff.SetDispatcher(d) + + armCtx, stop := context.WithCancel(ctx) + defer stop() + go sOff.runEmbedBackfillLoop(armCtx) + + time.Sleep(600 * time.Millisecond) + if got := pendingCount(t, pool); got != 1 { + t.Fatalf("pending = %d, want 1 (Interval=0 is the explicit off switch)", got) + } + }) +} diff --git a/go/internal/events/scheduler.go b/go/internal/events/scheduler.go index ee3e2686..20f754e5 100644 --- a/go/internal/events/scheduler.go +++ b/go/internal/events/scheduler.go @@ -782,6 +782,14 @@ func (s *Scheduler) Run(ctx context.Context) { embedMigrateTicker := time.NewTicker(embedMigrateInterval) defer embedMigrateTicker.Stop() + // The dedicated embed-backfill arm (tracking issue: force-push.me/damienmoon/ctx/issues/1): pending + // embeddings used to be backfilled only at the top of the dream loop, so + // with dream.enabled=false (the default) or Dream switched off at runtime + // nothing embedded them in the background. The arm runs regardless of + // Dream; the dream loop keeps its inline call, the tx-wrapped + // FOR UPDATE SKIP LOCKED pick makes two pickers safe (Welle-49). + go s.runEmbedBackfillLoop(ctx) + // Dream runs in its own goroutine(s) as continuous loop(s). // DreamParallelism workers all share the same DB; PickBlock's FOR UPDATE // SKIP LOCKED ensures distinct blocks per worker. Backfill stays single- @@ -2043,6 +2051,78 @@ func (t *txTail) done() { } } +// backfillArmPoll is how often the dedicated embed-backfill arm re-reads its +// config while embed_backfill.interval is 0 (arm off), so a hot re-enable +// takes effect without a restart (contract.recheck_interval convention). +const backfillArmPoll = 15 * time.Second + +// runEmbedBackfillLoop is the dedicated background arm for pending +// embeddings (tracking issue: force-push.me/damienmoon/ctx/issues/1). It is the same work as the call at the +// top of runDreamLoop — backfillOneEmbedding under the iterated tenant's +// snapshot and router — on its own cadence: a successful pick loops at once, +// an empty pick waits embed_backfill.interval, an error waits 10 s. Under +// !enforcing() it yields to interactive demand exactly like the dream loop +// (P-Fallback); under Enforcing the wire call waits at its target in its +// background lease. Interval <= 0 turns the arm off; it then polls the +// config every backfillArmPoll so a hot enable needs no restart. +func (s *Scheduler) runEmbedBackfillLoop(ctx context.Context) { + slog.Info("scheduler: embed backfill arm started", "poll", backfillArmPoll) + var tenantCursor uint64 + wait := func(d time.Duration) bool { + select { + case <-ctx.Done(): + return false + case <-time.After(d): + return true + } + } + for ctx.Err() == nil { + tenants := s.backgroundTenantsFn(ctx) + if len(tenants) == 0 { + tenants = []backgroundTenant{{scope: store.GlobalScope}} + } + bt := tenants[tenantCursor%uint64(len(tenants))] + tenantCursor++ + cfg := s.cfg.SnapshotForTenant(ctx, bt.scope) + + interval := cfg.EmbedBackfill.Interval + if interval <= 0 { + if !wait(backfillArmPoll) { + return + } + continue + } + for !s.enforcing() && s.interactiveDemand() > 0 { + if !wait(dreamYieldWait) { + return + } + } + + backfilled, err := s.backfillArmOnce(ctx, cfg, bt.scope) + switch { + case err != nil: + slog.Error("scheduler: embed backfill arm error", "error", err) + if !wait(10 * time.Second) { + return + } + case backfilled: + continue // Loop at once: more may be pending. + default: + if !wait(interval) { + return + } + } + } +} + +// backfillArmOnce is one pick of the dedicated arm with its own panic guard, +// so a panic in one cycle is logged and the arm goes on (the dream loop's +// runDreamCycle has the same shape). +func (s *Scheduler) backfillArmOnce(ctx context.Context, cfg *config.Config, tenant string) (bool, error) { + defer guardPanic("embed backfill arm") + return s.backfillOneEmbedding(ctx, s.newRouter(cfg, tenant), cfg) +} + // backfillOneEmbedding finds one block with missing embedding, generates it, and stores it. // Returns true if a block was backfilled, false if none needed. // The embed chains over the pool (G28): role dream-embed when configured, diff --git a/go/internal/handler/backends_manage.go b/go/internal/handler/backends_manage.go index ee1e5fda..50dbb7cc 100644 --- a/go/internal/handler/backends_manage.go +++ b/go/internal/handler/backends_manage.go @@ -799,10 +799,18 @@ func dataCollectionEscapeOn(b *backends.Backend) bool { func probeBackendURL(ctx context.Context, b *backends.Backend, checks map[string]string) bool { url := b.Host - switch b.ProviderClass { - case backends.ProviderLlamaCpp: + anthropic := b.Protocol == backends.ProtocolAnthropic + switch { + case anthropic: + // The Messages API root answers nothing useful; /v1/models is the + // cheapest keyed GET (same root convention: base_url + "/v1/..."). + // A bad key is a 401 here, reported as checks.auth below; the row + // still counts as reachable, so a probe:"chat" goes on to the + // 1-token chat, which 401s the same way — no tokens are billed. + url += "/v1/models" + case b.ProviderClass == backends.ProviderLlamaCpp: url += "/health" - case backends.ProviderOpenRouter: + case b.ProviderClass == backends.ProviderOpenRouter: // /v1/key matches the chat path convention (base_url is the API root // without the version segment, b.Host+"/v1/..."). url += "/v1/key" @@ -814,7 +822,13 @@ func probeBackendURL(ctx context.Context, b *backends.Backend, checks map[string checks["base_url"] = "error: invalid URL" return false } - if b.APIKey != "" { + switch { + case anthropic: + httpReq.Header.Set("anthropic-version", "2023-06-01") + if b.APIKey != "" { + httpReq.Header.Set("x-api-key", b.APIKey) + } + case b.APIKey != "": httpReq.Header.Set("Authorization", "Bearer "+b.APIKey) } resp, err := http.DefaultClient.Do(httpReq) //nolint:gosec // G704: admin-gated runtime data; reaching arbitrary URLs is this action's function diff --git a/go/internal/handler/backends_test_anthropic_test.go b/go/internal/handler/backends_test_anthropic_test.go new file mode 100644 index 00000000..8d871b70 --- /dev/null +++ b/go/internal/handler/backends_test_anthropic_test.go @@ -0,0 +1,68 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/GottZ/ctx/internal/backends" +) + +// TestBackendTestAnthropicProbe: an anthropic-protocol backend-test probes +// GET /v1/models with the Messages API headers (x-api-key + +// anthropic-version, no Bearer) and reports base_url/auth ok on a 2xx — the +// generic branch would GET the bare root with a Bearer header, which the +// API answers 404 (unreachable) or 401. +func TestBackendTestAnthropicProbe(t *testing.T) { + var mu sync.Mutex + var paths []string + var hdr http.Header + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, r.URL.Path) + hdr = r.Header.Clone() + mu.Unlock() + if r.URL.Path == "/v1/models" { + _, _ = w.Write([]byte(`{"data":[{"id":"claude-opus-5-5"}]}`)) + return + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + + bp := backends.NewPool(nil, nil) + bp.SeedSnapshotForTest([]backends.Backend{{ + ID: "0190-ant", Name: "anthropic", Host: srv.URL, + Protocol: backends.ProtocolAnthropic, ProviderClass: backends.ProviderGeneric, + Trust: backends.TrustNoCredentials, Locality: backends.LocalityExternal, + Roles: []string{backends.RoleSynthesis}, NumCtx: 100000, + ModelMap: map[string]backends.ModelSpec{"default": {Model: "claude-opus-5-5"}}, + APIKey: "sk-ant-test", APIKeyRef: "anthropic-chat", Enabled: true, + }}) + + rec := manageReqWithPool(t, adminAR(), bp, map[string]any{ + "action": "backend-test", "id": "0190-ant", + }) + if rec.Code != http.StatusOK { + t.Fatalf("backend-test: status %d (body %s)", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + for _, want := range []string{`"base_url":"ok"`, `"auth":"ok"`} { + if !strings.Contains(body, want) { + t.Errorf("backend-test response lacks %s: %s", want, body) + } + } + mu.Lock() + defer mu.Unlock() + if len(paths) != 1 || paths[0] != "/v1/models" { + t.Errorf("probe paths = %v, want exactly /v1/models", paths) + } + if hdr.Get("x-api-key") != "sk-ant-test" || hdr.Get("anthropic-version") == "" { + t.Errorf("probe headers = %v, want x-api-key + anthropic-version", hdr) + } + if hdr.Get("Authorization") != "" { + t.Errorf("probe sent a Bearer header on the anthropic wire: %q", hdr.Get("Authorization")) + } +} diff --git a/go/internal/handler/backfill_syncbudget_integration_test.go b/go/internal/handler/backfill_syncbudget_integration_test.go new file mode 100644 index 00000000..bcfba1b2 --- /dev/null +++ b/go/internal/handler/backfill_syncbudget_integration_test.go @@ -0,0 +1,230 @@ +//go:build integration + +// Rot-Gate for the tracking issue, force-push.me/damienmoon/ctx/issues/1: the query-path pre-search backfill +// (backfillPending) had a COUNT cap (embed_backfill.sync_cap) but no TIME +// budget. One block whose embed could not finish inside the caller's +// deadline consumed the entire deadline (live: a ~10k-token block against a +// CPU embed backend at ~23 tok/s needs ~7 min, the CLI client's deadline is +// 120 s), the question's own embed was never admitted ("embedcache: +// admission: context canceled") and the query answered 500. The failure was +// memoized as class `wire` — backend-down semantics for a backend that was +// healthy, just slow for this caller — so every query that landed after the +// backoff lapsed repeated the burn. +// +// Rot and Grün run through the SAME production function. SyncBudget=0 is BY +// CONSTRUCTION the historical shape (no deadline on the wire call, no +// budget check at the loop head); SyncBudget>0 is the fix. +// +// SyncBudget=0 (Ist): the slow block holds the loop for its full embed time. +// SyncBudget=300ms (Soll): the loop returns inside the budget, the block +// is memoized caller_timeout with next_attempt_at <= now(), the query +// path never picks it again, the background arm's predicate still does. +// +// Run: `go test -tags=integration ./internal/handler/ -run TestBackfillPending_SyncBudget -count=1 -v`. +package handler + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/GottZ/ctx/internal/blocktype" + "github.com/GottZ/ctx/internal/config" + "github.com/GottZ/ctx/internal/store" + "github.com/GottZ/ctx/internal/testdb" +) + +// slowEmbedServer answers like fakeEmbedServer, but only after `delay`, or +// at once with the request's cancellation if the client goes away first. +func slowEmbedServer(t *testing.T, delay time.Duration) (*httptest.Server, *atomic.Int32, *atomic.Int32) { + t.Helper() + var hits, canceled atomic.Int32 + vec := make([]float64, 1024) + for i := range vec { + vec[i] = float64((i % 2) * 2) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + // Drain the body first: net/http starts the background read that + // turns a client disconnect into r.Context() cancellation only once + // the request body has been consumed (registerOnHitEOF in + // readRequest). Without this the handler never sees the cancel. + _, _ = io.Copy(io.Discard, r.Body) + select { + case <-time.After(delay): + case <-r.Context().Done(): + canceled.Add(1) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float64{vec}}) + })) + t.Cleanup(srv.Close) + return srv, &hits, &canceled +} + +func TestBackfillPending_SyncBudget_Integration(t *testing.T) { + if testing.Short() { + t.Skip("integration test") + } + pool := testdb.SetupTestDB(t) + ctx := context.Background() + + const title = "syncbudget-slow-block" + seed := func(t *testing.T) { + t.Helper() + if _, err := pool.Exec(ctx, + `INSERT INTO context_blocks (category, title, content, scope, created_at, updated_at) + VALUES ('learnings', $1, 'body of the sync-budget fixture block', 'private', now(), now())`, title); err != nil { + t.Fatalf("seed block: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM context_embed_failures WHERE block_id IN (SELECT id FROM context_blocks WHERE title = $1)`, title) + _, _ = pool.Exec(ctx, `DELETE FROM context_blocks WHERE title = $1`, title) + }) + } + readMemo := func(t *testing.T) (class string, dueNow bool, attempts int) { + t.Helper() + err := pool.QueryRow(ctx, + `SELECT f.last_class, f.next_attempt_at <= now(), f.attempts + FROM context_embed_failures f JOIN context_blocks cb ON cb.id = f.block_id + WHERE cb.title = $1 AND f.migration_id IS NULL`, title).Scan(&class, &dueNow, &attempts) + if err != nil { + t.Fatalf("read memo: %v", err) + } + return class, dueNow, attempts + } + newHandler := func(t *testing.T, delay time.Duration) (*QueryHandler, *atomic.Int32, *atomic.Int32) { + t.Helper() + srv, hits, canceled := slowEmbedServer(t, delay) + st := &countingStore{} + st.cfg.Store(snapshotTestConfig()) + h := NewQueryHandler(pool, st, embedPool(srv.URL), nil, blocktype.NewRegistry(), snapshotTestAdmitter(t)) + return h, hits, canceled + } + + // Rot: SyncBudget=0 is the historical shape — the loop waits for the + // slow backend however long it takes. Nothing bounds the request's time. + t.Run("red_unbudgeted_waits_for_the_slow_backend", func(t *testing.T) { + seed(t) + h, hits, _ := newHandler(t, 1500*time.Millisecond) + cfg := &config.Config{EmbedBackfill: config.EmbedBackfillConfig{ + SyncCap: 4, SyncBudget: 0, MaxTokens: 1_000_000, BackoffBase: time.Minute, BackoffCap: time.Hour, + }} + + started := time.Now() + got := h.backfillPending(ctx, nil, "private", h.embedAdmission(), cfg) + elapsed := time.Since(started) + if got != 1 { + t.Fatalf("RED premise: backfilled = %d, want 1 (unbudgeted loop embeds the slow block)", got) + } + if elapsed < 1400*time.Millisecond { + t.Fatalf("RED premise: returned after %v, want >= 1.4 s (the loop waited for the backend)", elapsed) + } + if hits.Load() != 1 { + t.Errorf("wire hits = %d, want 1", hits.Load()) + } + t.Logf("RED confirmed: SyncBudget=0 held the request for %v on one slow block", elapsed) + }) + + // Grün: SyncBudget bounds the loop. The slow block is cut at the budget, + // memoized caller_timeout and due for the background arm at once; the + // parent context is untouched, so the question embed keeps its time. + t.Run("budget_cuts_the_slow_block_and_hands_it_to_the_arm", func(t *testing.T) { + seed(t) + h, hits, canceled := newHandler(t, 5*time.Second) + cfg := &config.Config{EmbedBackfill: config.EmbedBackfillConfig{ + SyncCap: 4, SyncBudget: 300 * time.Millisecond, MaxTokens: 1_000_000, BackoffBase: time.Minute, BackoffCap: time.Hour, + }} + reqCtx, cancelReq := context.WithCancel(ctx) + defer cancelReq() + + started := time.Now() + got := h.backfillPending(reqCtx, nil, "private", h.embedAdmission(), cfg) + elapsed := time.Since(started) + if got != 0 { + t.Fatalf("backfilled = %d, want 0 (the slow block must not count)", got) + } + if elapsed > 2*time.Second { + t.Fatalf("returned after %v, want well inside the 5 s backend delay (budget 300 ms)", elapsed) + } + if reqCtx.Err() != nil { + t.Fatalf("request context is dead (%v); the budget must cut only the embed call", reqCtx.Err()) + } + if hits.Load() != 1 { + t.Errorf("wire hits = %d, want 1", hits.Load()) + } + // The backend saw the cancellation: no 7-minute embed keeps burning CPU + // for a caller that is gone (the live task was cancelled at 30 %). + deadline := time.Now().Add(2 * time.Second) + for canceled.Load() == 0 && time.Now().Before(deadline) { + time.Sleep(20 * time.Millisecond) + } + if canceled.Load() != 1 { + t.Errorf("backend saw %d cancellations, want 1", canceled.Load()) + } + + class, dueNow, attempts := readMemo(t) + if class != string(store.EmbedFailureCallerTimeout) { + t.Errorf("memo class = %q, want %q", class, store.EmbedFailureCallerTimeout) + } + if !dueNow { + t.Errorf("memo next_attempt_at is in the future; a caller_timeout must be due for the background arm at once") + } + if attempts != 1 { + t.Errorf("memo attempts = %d, want 1", attempts) + } + + // Pfad A never picks it again, however much budget it has. + cfg.EmbedBackfill.SyncBudget = 10 * time.Second + h2, hits2, _ := newHandler(t, 0) + if again := h2.backfillPending(ctx, nil, "private", h2.embedAdmission(), cfg); again != 0 { + t.Fatalf("second backfillPending embedded %d, want 0 (caller_timeout blocks belong to the background arm)", again) + } + if hits2.Load() != 0 { + t.Errorf("second handler made %d wire calls, want 0", hits2.Load()) + } + + // The background arm's predicate (the shared one) still sees it. + var visibleToArm int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM context_blocks + WHERE embedding IS NULL AND NOT is_archived AND title = $1`+store.EmbedFailureExcludedPredicate, title). + Scan(&visibleToArm); err != nil { + t.Fatalf("arm predicate: %v", err) + } + if visibleToArm != 1 { + t.Errorf("background predicate sees %d block(s), want 1 (caller_timeout is due at once for the arm)", visibleToArm) + } + }) + + // A request that is cancelled mid-embed (client gone) is the same case + // for the block: caller_timeout, not wire — the backend did nothing wrong. + t.Run("request_cancel_is_caller_timeout_not_wire", func(t *testing.T) { + seed(t) + h, _, _ := newHandler(t, 5*time.Second) + cfg := &config.Config{EmbedBackfill: config.EmbedBackfillConfig{ + SyncCap: 4, SyncBudget: 10 * time.Second, MaxTokens: 1_000_000, BackoffBase: time.Minute, BackoffCap: time.Hour, + }} + reqCtx, cancelReq := context.WithCancel(ctx) + go func() { + time.Sleep(200 * time.Millisecond) + cancelReq() + }() + if got := h.backfillPending(reqCtx, nil, "private", h.embedAdmission(), cfg); got != 0 { + t.Fatalf("backfilled = %d, want 0", got) + } + class, dueNow, _ := readMemo(t) + if class != string(store.EmbedFailureCallerTimeout) { + t.Errorf("memo class = %q, want %q", class, store.EmbedFailureCallerTimeout) + } + if !dueNow { + t.Errorf("memo must be due at once for the background arm") + } + }) +} diff --git a/go/internal/handler/query.go b/go/internal/handler/query.go index 3ab3d61d..e18abbb5 100644 --- a/go/internal/handler/query.go +++ b/go/internal/handler/query.go @@ -729,7 +729,10 @@ func (h *QueryHandler) HandleQuery(w http.ResponseWriter, r *http.Request) { "request_id", requestID, ) } else if llm.DetectGerman(query) { - slog.Info("german detected, translating", + // The heuristic fires on umlauts and German stopwords, so Swedish, + // Danish and Finnish questions land here too; the translation to + // English is right for them as well, only the old label was not. + slog.Info("non-english heuristic matched, translating", "request_id", requestID, ) translatedQuery, err := llm.TranslateQuery(ctx, h.pool, h.backendPool, querySens, query, ar.ApiKeyID, h.admission()) @@ -1710,12 +1713,34 @@ func (h *QueryHandler) logAccess(ar *auth.AuthResult, results []rrf.SearchResult func (h *QueryHandler) backfillPending(ctx context.Context, floor config.ScopeFloor, scope string, adm embedcache.Admission, cfg *config.Config) int { count := 0 syncCap := cfg.EmbedBackfill.SyncCap + // Time budget next to the count cap (embed_backfill.sync_budget, issue + // force-push.me/damienmoon/ctx/issues/1): SyncCap bounds how MANY blocks a request embeds + // inline, this bounds how LONG. One block that cannot embed inside the + // caller's deadline used to consume the whole deadline — the question's + // own embed was then never admitted and the query answered 500. Every + // block's wire call now runs under what is left of the budget; a block + // that overruns it is memoized caller_timeout and left to the + // background arm (runEmbedBackfillLoop), and the loop ends so the + // question embed keeps the rest of the request. Zero = unbounded, the + // pre-budget shape (explicit opt-out, same convention as SyncCap). + var budgetDeadline time.Time + if budget := cfg.EmbedBackfill.SyncBudget; budget > 0 { + budgetDeadline = time.Now().Add(budget) + } for attempts := 0; syncCap <= 0 || attempts < syncCap; attempts++ { + if !budgetDeadline.IsZero() && !time.Now().Before(budgetDeadline) { + slog.Info("query backfill: sync budget spent, rest left to the background arm", + "budget", cfg.EmbedBackfill.SyncBudget, "backfilled", count) + break + } var blockID, title, content, sens, scope string + // Pfad A's predicate: the shared backoff exclusion PLUS every block + // already memoized caller_timeout (store.EmbedFailureQueryPath + // ExcludedPredicate) — such a block is the background arm's job. err := h.pool.QueryRow(ctx, `SELECT id, title, content, sensitivity, scope FROM context_blocks WHERE embedding IS NULL AND NOT is_archived`+ - store.EmbedFailureExcludedPredicate+store.RetrievalExcludedTypePredicate+` + store.EmbedFailureQueryPathExcludedPredicate+store.RetrievalExcludedTypePredicate+` LIMIT 1`).Scan(&blockID, &title, &content, &sens, &scope) if err != nil { break // No more pending blocks (or error). @@ -1758,9 +1783,17 @@ func (h *QueryHandler) backfillPending(ctx context.Context, floor config.ScopeFl // pool=nil: document embeddings land in the block row, not the cache // (today's semantics — the cache is for repeated query/keyword text). + // The wire call runs under the remaining sync budget; callerCut is + // read BEFORE cancelEmbed, which would make Err() non-nil by itself. + embedCtx, cancelEmbed := ctx, func() {} + if !budgetDeadline.IsZero() { + embedCtx, cancelEmbed = context.WithDeadline(ctx, budgetDeadline) + } vec, served, wireAttempts, wired, err := embedcache.EmbedChain( - ctx, nil, chain, backends.RoleEmbed, embedText, embed.PrefixDocument, + embedCtx, nil, chain, backends.RoleEmbed, embedText, embed.PrefixDocument, embedcache.ReportFunc(llm.PoolReporter(h.backendPool)), adm) + callerCut := err != nil && embedCtx.Err() != nil + cancelEmbed() if wired { // TENANT-DECISION(backfill-attribution): "" → NULL. backfillPending is // query-triggered but maintenance in nature — it embeds whatever blocks @@ -1782,7 +1815,20 @@ func (h *QueryHandler) backfillPending(ctx context.Context, floor config.ScopeFl // failure most often means the backend itself is unavailable, // and hammering it for the rest of the sync-cap budget within // one request helps nobody. - class, normalized := store.ClassifyEmbedError(err) + // A budget or request deadline is a statement about THIS caller, + // not about the backend: class caller_timeout, which the query + // path never re-picks and the background arm picks at once. + // Everything else keeps the wire/oversize classification. + var ( + class store.EmbedFailureClass + normalized string + ) + if callerCut { + class = store.EmbedFailureCallerTimeout + normalized = store.NormalizeEmbedError(class, err.Error()) + } else { + class, normalized = store.ClassifyEmbedError(err) + } // The memo write must survive the request context: the wire // failure above IS frequently the request deadline itself // (canceled context), and a memo lost to that same cancellation diff --git a/go/internal/llm/anthropic.go b/go/internal/llm/anthropic.go new file mode 100644 index 00000000..2ec4564b --- /dev/null +++ b/go/internal/llm/anthropic.go @@ -0,0 +1,374 @@ +// Native Anthropic Messages API wire (backends.ProtocolAnthropic): POST +// {base_url}/v1/messages with x-api-key + anthropic-version, the system +// prompt as a top-level cached block, and NO sampling parameters. +// +// Why a third wire instead of the openai path against Anthropic's +// OpenAI-compatible layer: that layer rejects every sampling parameter on +// the Claude 5 models (400 "temperature is deprecated for this model" — and +// chatOpenAI always sends temperature), drops response_format silently, has +// no prompt caching and no effort control, and Anthropic documents it as a +// comparison surface, not a production one. The ku-heimdall install ran a +// stripping sidecar in front of it for exactly those reasons; this file is +// what makes that sidecar unnecessary. +// +// Model-side facts this wire is built on (claude-api reference, 2026-10, +// Claude Opus 5.5 / Sonnet 5.5 / Fable 5.1 family): +// - temperature / top_p / top_k → 400: none of Options' sampling fields +// are ever marshalled; num_ctx stays a ctx-internal prompt budget. +// - thinking cannot be disabled (Opus 5.5: {type:"disabled"} is a 400 at +// every effort); omitting the field runs adaptive thinking. The +// model_map `think` toggle is therefore ignored here, and depth is +// steered with output_config.effort instead (resolveAnthropicEffort). +// - forced tool_choice (any/tool) → 400: never sent (auto is the default, +// matching buildStreamBody's doctrine on the openai wire). +// - output_config.format needs a JSON schema with additionalProperties: +// false on every object; ctx's "json" format carries no schema and the +// dream stages answer with arrays, so ChatJSON is a documented no-op on +// this wire — the pipelines' local parsers stay the validator, exactly +// the CTX_DREAM_JSON_MODE=off contract (docs/operations.md). +// - max_tokens is REQUIRED and thinking tokens count against it. +// - usage.input_tokens is the UNCACHED remainder; the total prompt is +// input + cache_read + cache_creation (anthropicUsageTotals), which is +// what PromptTokens reports so llmlog's prompt_tokens stays comparable +// with the other wires. The two cache counters travel separately for +// the llmlog metadata (ApplyProviderTelemetry). +// - 529 overloaded_error is a plain non-2xx: httpx.StatusError carries it +// into backends.Classify as ClassGateway (failover), 429 as +// ClassRateLimit with the Retry-After cooldown, 401 as ClassAuth. The +// same types inside a mid-stream error event classify the same way +// (StreamError.ProviderErrorType). +// - The wire targets the Claude 5 family (claude-opus-5-5 in production). +// output_config.effort is sent on every call; models that predate it +// (Sonnet 4.5, Haiku 4.5) reject the field, and on Opus 4.7/4.8 an +// omitted thinking field means NO thinking — neither is a supported row. +// +// Raw net/http rather than the official Go SDK, like every other ctx wire: +// the backend pool, the dispatch invariant (one pinned +// http.NewRequestWithContext per wire file, chat/dispatch_invariant_test.go) +// and the failover classifier all key on ctx's own request/response path, +// and the request shape is small enough that an SDK would add a dependency +// without removing any of that. +package llm + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/httpx" + "github.com/GottZ/ctx/internal/promptguard" +) + +const ( + // anthropicVersion is the Messages API version header. 2023-06-01 is the + // only GA value; features ship behind anthropic-beta, not new versions. + anthropicVersion = "2023-06-01" + // anthropicDefaultMaxTokens applies when the caller sets no NumPredict: + // the field is required on this wire, and every production Options + // preset sets one (synthesis 500, translate 100, probe 1), so this is + // the safety net for a bare Options{}, sized so a long answer is not + // cut mid-thought (claude-api reference default for non-streaming). + anthropicDefaultMaxTokens = 16000 + // anthropicEffortKey is the model_map params / extra_body key that is + // LIFTED into output_config.effort instead of being merged top-level. + anthropicEffortKey = "effort" +) + +// --- request shape ---. + +type anthropicCacheControl struct { + Type string `json:"type"` // "ephemeral" +} + +// anthropicTextBlock is one text content block; CacheControl marks the +// prefix breakpoint (the system prompt here). +type anthropicTextBlock struct { + Type string `json:"type"` + Text string `json:"text"` + CacheControl *anthropicCacheControl `json:"cache_control,omitempty"` +} + +// anthropicMessage is one turn. Content is either a plain string (the +// single-user-prompt pipelines) or a []any of content blocks (the chat +// harness: text, tool_use, tool_result). +type anthropicMessage struct { + Role string `json:"role"` + Content any `json:"content"` +} + +type anthropicOutputConfig struct { + Effort string `json:"effort"` +} + +// anthropicTool is the Messages API tool definition (input_schema, not the +// OpenAI function envelope). +type anthropicTool struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + InputSchema json.RawMessage `json:"input_schema"` +} + +type anthropicRequest struct { + Model string `json:"model"` + MaxTokens int `json:"max_tokens"` + System []anthropicTextBlock `json:"system,omitempty"` + Messages []anthropicMessage `json:"messages"` + Stream bool `json:"stream,omitempty"` + OutputConfig anthropicOutputConfig `json:"output_config"` + Tools []anthropicTool `json:"tools,omitempty"` + // CacheControl at the top level is the API's automatic breakpoint on the + // last cacheable block — set on the multi-turn stream path only. + CacheControl *anthropicCacheControl `json:"cache_control,omitempty"` +} + +// --- response shape ---. + +type anthropicUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheReadInputTokens int `json:"cache_read_input_tokens"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens"` +} + +// anthropicContentBlock is one response block. text carries Text; +// tool_use carries ID/Name/Input; thinking blocks are skipped (their text +// is empty under the default display anyway). +type anthropicContentBlock struct { + Type string `json:"type"` + Text string `json:"text"` + ID string `json:"id"` + Name string `json:"name"` + Input json.RawMessage `json:"input"` +} + +type anthropicResponse struct { + ID string `json:"id"` + Model string `json:"model"` + StopReason string `json:"stop_reason"` + Content []anthropicContentBlock `json:"content"` + Usage anthropicUsage `json:"usage"` +} + +// anthropicError is the body of every non-2xx answer and of a mid-stream +// error event: {"type":"error","error":{"type":"...","message":"..."}}. +type anthropicError struct { + Error struct { + Type string `json:"type"` + Message string `json:"message"` + } `json:"error"` +} + +// --- helpers shared by the non-streaming and the streaming path ---. + +// resolveAnthropicEffort picks output_config.effort: the role's model_map +// params (arriving in Options.Extra through applyModelParams' default +// branch) beat the row's extra_body, which beats the explicit default. +// Validation (backends.validateAnthropic) has already bounded both carriers +// to the accepted set; a non-string value here falls through to the next +// carrier rather than reaching the wire. +func resolveAnthropicEffort(opts Options, b *backends.Backend) string { + if v, ok := opts.Extra[anthropicEffortKey].(string); ok && v != "" { + return v + } + if v, ok := b.ExtraBody[anthropicEffortKey].(string); ok && v != "" { + return v + } + return backends.AnthropicDefaultEffort +} + +func anthropicMaxTokens(opts Options) int { + if opts.NumPredict > 0 { + return opts.NumPredict + } + return anthropicDefaultMaxTokens +} + +// promptguardRuleMarker is the opening of promptguard.Rule's sentence — the +// per-call nonce rule every pipeline appends to its system prompt. +var promptguardRuleMarker = "Everything between <" + promptguard.GuardTag + " id=" + +// anthropicSystem renders the system prompt as the cached top-level block. +// An empty prompt yields no system field (the API rejects an empty text +// block). +// +// The cache is a byte-exact prefix match, and the pipelines end their +// system prompt with promptguard.Rule(nonce) — a fresh nonce per call. One +// block carrying both would be written to the cache on every call and read +// back never (a pure 1.25x surcharge, visible as cache_creation > 0 with +// cache_read == 0 on every row). So the rule is split off into a second, +// unmarked text block: the stable prefix before it is the breakpoint. +func anthropicSystem(systemPrompt string) []anthropicTextBlock { + if systemPrompt == "" { + return nil + } + if i := strings.LastIndex(systemPrompt, promptguardRuleMarker); i > 0 { + return []anthropicTextBlock{ + {Type: "text", Text: systemPrompt[:i], CacheControl: &anthropicCacheControl{Type: "ephemeral"}}, + {Type: "text", Text: systemPrompt[i:]}, + } + } + return []anthropicTextBlock{{ + Type: "text", Text: systemPrompt, + CacheControl: &anthropicCacheControl{Type: "ephemeral"}, + }} +} + +// buildAnthropicBody marshals the request and merges the two extra carriers +// — Options.Extra (model_map params) first, b.ExtraBody last, so a backend +// row's extra_body keeps the final word exactly as on the openai wire +// (applyOpenAIBodyExtras). The effort key is consumed by +// resolveAnthropicEffort and never merged top-level (it is not a Messages +// API field). Everything else merges verbatim: that is the operator's +// escape hatch for fields this wire does not model (thinking.display, +// stop_sequences, metadata.user_id, …). +func buildAnthropicBody(req anthropicRequest, opts Options, b *backends.Backend) ([]byte, error) { + body, err := json.Marshal(req) + if err != nil { + return nil, err + } + extra := make(map[string]any, len(opts.Extra)+len(b.ExtraBody)) + for k, v := range opts.Extra { + if k != anthropicEffortKey { + extra[k] = v + } + } + for k, v := range b.ExtraBody { + if k != anthropicEffortKey { + extra[k] = v + } + } + return mergeJSONFields(body, extra) +} + +// setAnthropicHeaders mirrors setChatHeaders' doctrine: ExtraHeaders first, +// the core headers LAST so a row edited past the credential-carrier +// denylist can never override the key derived from api_key_ref or the +// version pin. +func setAnthropicHeaders(req *http.Request, b *backends.Backend) { + for k, v := range b.ExtraHeaders { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("anthropic-version", anthropicVersion) + if b.APIKey != "" { + req.Header.Set("x-api-key", b.APIKey) + } +} + +// mapAnthropicStopReason translates the Messages API vocabulary into the +// one every FinishReason consumer already compares against (dream's cap +// detection reads "length", the chat harness reads "tool_calls"): a raw +// pass-through — the doctrine for the ollama/openai pair, whose +// vocabularies overlap — would make an Anthropic cap hit invisible to +// evaluate.go and a tool call look like a final answer to engine.go. Unknown +// values pass through raw so a new provider state is observable in llmlog; +// an absent reason stays "" (unknown), never a fabricated "stop". +func mapAnthropicStopReason(s string) string { + switch s { + case "": + return "" + case "end_turn", "stop_sequence", "pause_turn": + return "stop" + case "max_tokens", "model_context_window_exceeded": + return "length" + case "tool_use": + return "tool_calls" + case "refusal": + return "content_filter" + default: + return s + } +} + +// anthropicUsageTotals folds the usage object into ctx's counters: the +// prompt total (uncached remainder + cache read + cache write) and the +// output count, plus the two cache counters for the llmlog metadata. +func anthropicUsageTotals(u anthropicUsage) (prompt, output, cacheRead, cacheCreate int) { + return u.InputTokens + u.CacheReadInputTokens + u.CacheCreationInputTokens, + u.OutputTokens, u.CacheReadInputTokens, u.CacheCreationInputTokens +} + +// anthropicStatusError wraps a non-2xx answer the way the other wires do +// (httpx.StatusError for the failover classifier, rendered string unchanged) +// and prefixes the provider's error type when the body parses, so a log +// line reads "overloaded_error: Overloaded" instead of raw JSON. +func anthropicStatusError(resp *http.Response) error { + errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + var ae anthropicError + if json.Unmarshal(errBody, &ae) == nil && ae.Error.Type != "" { + errBody = []byte(ae.Error.Type + ": " + ae.Error.Message) + } + return fmt.Errorf("llm: %w", httpx.NewStatusError(resp, errBody)) +} + +// --- non-streaming path ---. + +// chatAnthropic is the ProtocolAnthropic leg of chatWithFormat. format is +// accepted for signature parity and ignored (see the package comment on +// structured outputs); think is ignored (thinking cannot be disabled). +func chatAnthropic(ctx context.Context, b backends.Backend, systemPrompt, userPrompt string, opts Options, _ string, timeout time.Duration) (*ChatResponse, error) { + req := anthropicRequest{ + Model: b.Model, + MaxTokens: anthropicMaxTokens(opts), + System: anthropicSystem(systemPrompt), + Messages: []anthropicMessage{{Role: "user", Content: userPrompt}}, + OutputConfig: anthropicOutputConfig{Effort: resolveAnthropicEffort(opts, &b)}, + } + body, err := buildAnthropicBody(req, opts, &b) + if err != nil { + return nil, fmt.Errorf("llm: marshal request: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, b.Host+"/v1/messages", bytes.NewReader(body)) //nolint:gosec // G704: host from the F3 backend pool (context_backends) — admin-gated + locality-validated, no free user input. + if err != nil { + return nil, fmt.Errorf("llm: create request: %w", err) + } + setAnthropicHeaders(httpReq, &b) + + resp, err := httpx.DoRetryOnce(httpClient, httpReq, body) + if err != nil { + return nil, fmt.Errorf("llm: request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, anthropicStatusError(resp) + } + + var result anthropicResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("llm: decode response: %w", err) + } + + var text strings.Builder + for _, blk := range result.Content { + if blk.Type == "text" { + text.WriteString(blk.Text) + } + } + prompt, output, cacheRead, cacheCreate := anthropicUsageTotals(result.Usage) + return &ChatResponse{ + Message: Message{Role: "assistant", Content: text.String()}, + EvalCount: output, + PromptTokens: prompt, + CacheReadTokens: cacheRead, + CacheCreationTokens: cacheCreate, + FinishReason: mapAnthropicStopReason(result.StopReason), + // The message id (msg_…) is the audit handle that exists on both the + // stream and the non-stream path (the request-id header is the one + // support quotes; it is not read here). The model column stays + // row-faithful — Anthropic echoes the requested id, there is no + // models-fallback to record — and no cost is reported on this wire + // (billing is read in the console). + ProviderRequestID: result.ID, + }, nil +} diff --git a/go/internal/llm/anthropic_live_test.go b/go/internal/llm/anthropic_live_test.go new file mode 100644 index 00000000..ded6200f --- /dev/null +++ b/go/internal/llm/anthropic_live_test.go @@ -0,0 +1,212 @@ +// Live smoke of the anthropic wire against api.anthropic.com. Skipped under +// -short and without CTX_ANTHROPIC_SMOKE_KEY, so CI never spends tokens — +// note the integration job runs WITHOUT -short, so there the gate is the +// absent variable alone (nothing in this file does work before the skip). +// Run it by hand from a shell that holds the key: +// +// CTX_ANTHROPIC_SMOKE_KEY=sk-ant-… go test ./internal/llm/ -run Live -count=1 -v +// +// It proves what the fakes cannot: the request shape is accepted by the real +// endpoint, the prompt cache hits on the second call, a tool round trip +// completes (replaying the thinking block when the model emits one — it +// logs when it did not), effort is accepted, the 1-token probe's behaviour, +// and the 401 classification. Each probe logs the usage it saw. Cost: a few +// thousand tokens per run. +package llm + +import ( + "context" + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/httpx" + "github.com/GottZ/ctx/internal/promptguard" +) + +func liveAnthropicBackend(t *testing.T) backends.Backend { + t.Helper() + if testing.Short() { + t.Skip("live smoke skipped under -short") + } + key := os.Getenv("CTX_ANTHROPIC_SMOKE_KEY") + if key == "" { + t.Skip("CTX_ANTHROPIC_SMOKE_KEY not set") + } + model := os.Getenv("CTX_ANTHROPIC_SMOKE_MODEL") + if model == "" { + model = "claude-opus-5-5" + } + host := os.Getenv("CTX_ANTHROPIC_SMOKE_HOST") + if host == "" { + host = "https://api.anthropic.com" + } + return backends.Backend{ + Host: host, Protocol: backends.ProtocolAnthropic, ProviderClass: backends.ProviderGeneric, + Model: model, Trust: backends.TrustNoCredentials, Locality: backends.LocalityExternal, + APIKey: key, APIKeyRef: "smoke", NumCtx: 100000, + ModelMap: map[string]backends.ModelSpec{"default": {Model: model}}, + } +} + +// liveSystemPrompt is a stable prefix comfortably above the 512-token cache +// minimum, followed by the per-call promptguard rule exactly as the +// pipelines send it. +func liveSystemPrompt() string { + var sb strings.Builder + sb.WriteString("You are ctx, a knowledge store assistant. Answer briefly and only from the sources given. ") + for i := 0; i < 60; i++ { + sb.WriteString("Rule ") + sb.WriteString(strings.Repeat("alpha beta gamma delta ", 3)) + sb.WriteString("applies to every answer; cite the source id in brackets. ") + } + sb.WriteString(promptguard.Rule(promptguard.NewNonce())) + return sb.String() +} + +func TestAnthropicLiveNonStreamAndCache(t *testing.T) { + b := liveAnthropicBackend(t) + ctx := context.Background() + opts := SynthesisOptions(100000) // the production preset: temperature, repeat penalty (both dropped), NumPredict 500 + first, err := Chat(ctx, b, liveSystemPrompt(), "Reply with exactly: ok", opts, 60*time.Second) + if err != nil { + t.Fatalf("first call: %v", err) + } + t.Logf("first: finish=%q out=%d prompt=%d cache_read=%d cache_create=%d id=%s content=%q", + first.FinishReason, first.EvalCount, first.PromptTokens, first.CacheReadTokens, first.CacheCreationTokens, first.ProviderRequestID, first.Message.Content) + if first.Message.Content == "" || first.EvalCount == 0 { + t.Errorf("empty answer: %+v", first) + } + second, err := Chat(ctx, b, liveSystemPrompt(), "Reply with exactly: ok again", opts, 60*time.Second) + if err != nil { + t.Fatalf("second call: %v", err) + } + t.Logf("second: finish=%q out=%d prompt=%d cache_read=%d cache_create=%d", + second.FinishReason, second.EvalCount, second.PromptTokens, second.CacheReadTokens, second.CacheCreationTokens) + if second.CacheReadTokens == 0 { + t.Errorf("no cache hit on the second call with a byte-identical stable prefix (fresh nonce only): cache_read=0, cache_create=%d", second.CacheCreationTokens) + } + + // JSON mode is a no-op on this wire: the same call must still succeed. + j, err := ChatJSON(ctx, b, "Answer with a JSON object {\"ok\": true}.", "Go.", Options{NumPredict: 200}, 60*time.Second) + if err != nil { + t.Fatalf("ChatJSON: %v", err) + } + t.Logf("json-mode no-op: finish=%q content=%q", j.FinishReason, j.Message.Content) +} + +func TestAnthropicLiveEffortAndProbe(t *testing.T) { + b := liveAnthropicBackend(t) + ctx := context.Background() + for _, eff := range []string{"low", "high"} { + r, err := Chat(ctx, b, "Answer in one word.", "What colour is the sky on a clear day?", + Options{NumPredict: 300, Extra: map[string]any{"effort": eff}}, 60*time.Second) + if err != nil { + t.Fatalf("effort %s: %v", eff, err) + } + t.Logf("effort=%s finish=%q out=%d content=%q", eff, r.FinishReason, r.EvalCount, r.Message.Content) + } + // The backend-test chat probe shape: max_tokens 1 on a model that thinks. + p, err := Chat(ctx, b, "You are a connectivity probe.", "Reply with: ok", Options{Temperature: 0, NumPredict: 1}, 30*time.Second) + if err != nil { + var se *httpx.StatusError + if errors.As(err, &se) { + t.Logf("probe max_tokens=1: HTTP %d %s (the probe would report %s)", se.Code, se.Body, backends.Classify(err, backends.ProviderGeneric)) + } else { + t.Fatalf("probe: %v", err) + } + } else { + t.Logf("probe max_tokens=1: finish=%q out=%d content=%q", p.FinishReason, p.EvalCount, p.Message.Content) + } +} + +func TestAnthropicLiveStreamToolRoundTrip(t *testing.T) { + b := liveAnthropicBackend(t) + ctx := context.Background() + tool := ToolDef{Type: "function", Function: ToolDefFunction{ + Name: "get_weather", Description: "Current weather for a city", + Parameters: json.RawMessage(`{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}`), + }} + // A short tool turn may come back without any thinking block (adaptive + // thinking skips trivial turns — observed live 2026-10-11), which would + // leave the replay path unexercised. Ask for visible reasoning and a + // question that needs a step of deliberation before the call. + b.ExtraBody = map[string]any{"thinking": map[string]any{"type": "adaptive", "display": "summarized"}} + msgs := []ChatMsg{ + {Role: "system", Content: "You are a weather assistant. Use the get_weather tool to answer weather questions; never guess. Think about which city the user means before calling the tool."}, + {Role: "user", Content: "I am in the capital of Germany. Is it warm enough for a t-shirt outside right now? Use the tool."}, + } + var events []StreamEvent + res, err := ChatStreamBackend(ctx, b, b.Model, msgs, []ToolDef{tool}, Options{NumPredict: 2048, Extra: map[string]any{"effort": "high"}}, 90*time.Second, collectEvents(&events)) + if err != nil { + t.Fatalf("first stream: %v", err) + } + t.Logf("first stream: finish=%q calls=%d blocks=%d out=%d prompt=%d cache_read=%d cache_create=%d content=%q", + res.FinishReason, len(res.ToolCalls), len(res.ProviderBlocks), res.CompletionTokens, res.PromptTokens, res.CacheReadTokens, res.CacheCreationTokens, res.Content) + if res.FinishReason != "tool_calls" || len(res.ToolCalls) == 0 { + t.Fatalf("expected a tool call, got finish=%q calls=%+v", res.FinishReason, res.ToolCalls) + } + for _, blk := range res.ProviderBlocks { + t.Logf("provider block: %.120s", blk) + } + call := res.ToolCalls[0] + args := argsAsMap(t, call) + t.Logf("tool call: id=%s name=%s args=%v", call.ID, call.Function.Name, args) + + // Round trip: the assistant turn WITH its thinking block, then the result. + msgs = append(msgs, + ChatMsg{Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls, ProviderBlocks: res.ProviderBlocks}, + ChatMsg{Role: "tool", Content: `{"temperature_c": 21, "sky": "clear"}`, ToolCallID: call.ID}, + ) + events = events[:0] + // Same effort as the first iteration: a changed effort invalidates the + // prompt cache by itself and would mask the prefix question below. + final, err := ChatStreamBackend(ctx, b, b.Model, msgs, []ToolDef{tool}, Options{NumPredict: 2048, Extra: map[string]any{"effort": "high"}}, 90*time.Second, collectEvents(&events)) + if err != nil { + t.Fatalf("second stream (thinking replay): %v", err) + } + t.Logf("second stream: finish=%q out=%d prompt=%d cache_read=%d cache_create=%d content=%q", + final.FinishReason, final.CompletionTokens, final.PromptTokens, final.CacheReadTokens, final.CacheCreationTokens, final.Content) + if final.FinishReason != "stop" || !strings.Contains(final.Content, "21") { + t.Errorf("expected a final answer quoting the tool result, got finish=%q content=%q", final.FinishReason, final.Content) + } + if final.CacheReadTokens == 0 { + t.Logf("note: no cache read on the second iteration (prefix below the 512-token minimum is expected for this tiny prompt)") + } + if len(res.ProviderBlocks) == 0 { + t.Logf("REPLAY PATH NOT EXERCISED: the first turn carried no thinking block, so the round trip above and the one below are identical requests") + return + } + + // The same round trip WITHOUT the thinking block is what the wire did + // before the review fix — record what the API says about a STRIPPED + // block today (only meaningful when one was produced, see above). + noThinking := append([]ChatMsg{}, msgs[:2]...) + noThinking = append(noThinking, + ChatMsg{Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls}, + ChatMsg{Role: "tool", Content: `{"temperature_c": 21, "sky": "clear"}`, ToolCallID: call.ID}, + ) + if _, err := ChatStreamBackend(ctx, b, b.Model, noThinking, []ToolDef{tool}, Options{NumPredict: 256, Extra: map[string]any{"effort": "high"}}, 90*time.Second, nil); err != nil { + t.Logf("round trip with the produced thinking block STRIPPED: rejected — %v (the reference only asks to keep it; this records what the API does)", err) + } else { + t.Logf("round trip with the produced thinking block STRIPPED: accepted by the API today") + } +} + +func TestAnthropicLiveBadKeyClassifiesAuth(t *testing.T) { + b := liveAnthropicBackend(t) + b.APIKey = "sk-ant-invalid-smoke" + _, err := Chat(context.Background(), b, "s", "Reply with: ok", Options{NumPredict: 5}, 30*time.Second) + if err == nil { + t.Fatal("a bad key must fail") + } + cls := backends.Classify(err, backends.ProviderGeneric) + t.Logf("bad key: %v → %s", err, cls) + if cls != backends.ClassAuth { + t.Errorf("Classify = %s, want auth", cls) + } +} diff --git a/go/internal/llm/anthropic_stream.go b/go/internal/llm/anthropic_stream.go new file mode 100644 index 00000000..1c72d111 --- /dev/null +++ b/go/internal/llm/anthropic_stream.go @@ -0,0 +1,546 @@ +// Streaming chat with tool use over the native Anthropic Messages API +// (POST /v1/messages, stream:true) — the ProtocolAnthropic leg of +// ChatStreamBackend, feeding the SAME StreamEvent/StreamResult contract the +// web-chat harness (chat/engine.go) consumes from the openai wire. +// +// Wire shape (claude-api reference, 2026-10): every data frame carries its +// own "type", so the parser keys on that and ignores the SSE event: lines +// (same information, and keying on the payload survives a proxy that drops +// the event field). Frames handled: +// - message_start → usage (input + cache counters; the prompt total) +// - content_block_start→ tool_use: new pending call, ToolCallName event; +// text/thinking: nothing +// - content_block_delta→ text_delta: ContentDelta; input_json_delta: +// arguments fragment; thinking_delta / +// signature_delta: accumulated into the replay +// block, never surfaced to the sink +// - message_delta → stop_reason (mapped, FinishReason event) and +// usage.output_tokens +// - message_stop → done +// - error → *StreamError (HTTP 200, mid-stream — the same +// class OpenRouter reports) +// - ping → ignored +// +// Message mapping ([]ChatMsg → Messages API): system rows lift into the +// cached top-level system block; an assistant row with tool calls becomes +// [text?, tool_use…] blocks; a tool row becomes a user turn carrying a +// tool_result block, and consecutive same-role turns fold into ONE message +// (the API requires user/assistant alternation — the harness appends one +// tool row per call, so a parallel-call turn yields several tool rows in +// a row). ToolDef (OpenAI function envelope) becomes {name, description, +// input_schema}. tool_choice is never sent (forced choice is a 400 on +// Claude Opus 5.5; auto is also what buildStreamBody sends). +package llm + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/httpx" +) + +// --- request mapping ---. + +// anthropicToolUseBlock / anthropicToolResultBlock / anthropicTextBlock are +// the content blocks of the chat harness's turns. +type anthropicToolUseBlock struct { + Type string `json:"type"` // "tool_use" + ID string `json:"id"` + Name string `json:"name"` + Input json.RawMessage `json:"input"` +} + +type anthropicToolResultBlock struct { + Type string `json:"type"` // "tool_result" + ToolUseID string `json:"tool_use_id"` + Content string `json:"content"` + IsError bool `json:"is_error,omitempty"` +} + +// anthropicThinkingBlock / anthropicRedactedThinkingBlock are the reasoning +// blocks the stream parser re-assembles so the harness can replay them +// unchanged. The claude-api reference asks that an assistant turn whose +// tool_use still awaits its tool_result keep its thinking block, and the +// preserved-thinking check rejects a block that was ALTERED; the text is +// empty under the default display and the signature carries the +// reasoning. Live (claude-opus-5-5, 2026-10-11) a short tool-call turn +// came back with no thinking block at all and the block-less round trip +// was accepted — so this is the reference's rule honoured, not a 400 +// observed (what the API does with a STRIPPED block was not exercised). +type anthropicThinkingBlock struct { + Type string `json:"type"` // "thinking" + Thinking string `json:"thinking"` + Signature string `json:"signature"` +} + +type anthropicRedactedThinkingBlock struct { + Type string `json:"type"` // "redacted_thinking" + Data string `json:"data"` +} + +// orphanToolResult is the synthetic answer for a tool_use the history never +// answered (the turn was interrupted between the model's call and the +// tool's result — client gone, context cancelled). The API rejects an +// assistant tool_use that is not followed by its tool_result, and the +// orphan sits in the persisted history for the rest of the session, so +// without this repair every later turn of that session would 400. +const orphanToolResult = "no result recorded: the turn was interrupted before this tool ran" + +// anthropicMessages converts the multi-turn history: the joined system text +// and the alternating message list. Empty assistant turns without tool +// calls are dropped (the API rejects empty content), and an assistant turn +// that would end the list is kept — the harness never ends on one (it +// appends tool results), and a trailing assistant turn is a prefill the +// Claude 5 models reject with a clear 400 rather than something to mask. +// +// Every system row is lifted into the top-level system block — the harness +// puts exactly one, first. A mid-history system row would be moved ahead of +// the conversation (and invalidate the cached prefix); if the harness ever +// injects such rows they belong in messages as role:system turns instead. +func anthropicMessages(msgs []ChatMsg) (system string, out []anthropicMessage) { + var sys []string + appendBlocks := func(role string, blocks ...any) { + if n := len(out); n > 0 && out[n-1].Role == role { + existing, _ := out[n-1].Content.([]any) + out[n-1].Content = append(existing, blocks...) + return + } + out = append(out, anthropicMessage{Role: role, Content: blocks}) + } + // Tool-call ids are mandatory on this wire (tool_use.id and the + // matching tool_result.tool_use_id). A history served earlier by an + // openai-wire row can carry EMPTY ids — llama.cpp streams calls without + // one and the openai parser tolerates it — so empty ids are repaired + // here: each nameless call gets a synthetic id, and the tool rows that + // follow with an empty ToolCallID consume those ids in order (the + // harness appends one tool row per call, in call order). open holds the + // ids of the last assistant turn's calls that no tool row has answered + // yet; whatever is still open when the next turn starts (or the history + // ends) gets the synthetic error result. + var pendingIDs, open []string + synthetic := 0 + flushOrphans := func() { + for _, id := range open { + appendBlocks("user", anthropicToolResultBlock{Type: "tool_result", ToolUseID: id, Content: orphanToolResult, IsError: true}) + } + open = open[:0] + } + for _, m := range msgs { + switch m.Role { + case "system": + if m.Content != "" { + sys = append(sys, m.Content) + } + case "user": + flushOrphans() + appendBlocks("user", anthropicTextBlock{Type: "text", Text: m.Content}) + case "assistant": + flushOrphans() + var blocks []any + // Reasoning blocks first, verbatim — the order the API produced + // them in (thinking precedes the tool_use it introduced). + for _, raw := range m.ProviderBlocks { + blocks = append(blocks, raw) + } + if m.Content != "" { + blocks = append(blocks, anthropicTextBlock{Type: "text", Text: m.Content}) + } + pendingIDs = pendingIDs[:0] + for _, tc := range m.ToolCalls { + id := tc.ID + if id == "" { + synthetic++ + id = fmt.Sprintf("ctxcall_%d", synthetic) + pendingIDs = append(pendingIDs, id) + } + open = append(open, id) + blocks = append(blocks, anthropicToolUseBlock{ + Type: "tool_use", ID: id, Name: tc.Function.Name, + Input: anthropicToolInput(tc.Function.Arguments), + }) + } + if len(blocks) == 0 { + continue + } + appendBlocks("assistant", blocks...) + case "tool": + id := m.ToolCallID + if id == "" && len(pendingIDs) > 0 { + id, pendingIDs = pendingIDs[0], pendingIDs[1:] + } + for i, o := range open { + if o == id { + open = append(open[:i], open[i+1:]...) + break + } + } + appendBlocks("user", anthropicToolResultBlock{Type: "tool_result", ToolUseID: id, Content: m.Content}) + } + } + flushOrphans() + return strings.Join(sys, "\n\n"), out +} + +// anthropicToolInput renders assembled tool-call arguments as the tool_use +// input object. ToolCall carries the object form already (normalizeArgs); +// anything that is not a valid JSON object becomes {} — the executor has +// already answered such a call with an ok:false result, and the history +// must still replay. +func anthropicToolInput(args json.RawMessage) json.RawMessage { + t := bytes.TrimSpace(args) + if len(t) > 0 && t[0] == '{' && json.Valid(t) { + return t + } + return json.RawMessage("{}") +} + +// anthropicTools converts the OpenAI function envelopes. input_schema is +// required; a tool without parameters gets the empty object schema. +func anthropicTools(tools []ToolDef) []anthropicTool { + if len(tools) == 0 { + return nil + } + out := make([]anthropicTool, 0, len(tools)) + for _, t := range tools { + schema := bytes.TrimSpace(t.Function.Parameters) + if len(schema) == 0 || bytes.Equal(schema, []byte("null")) { + schema = []byte(`{"type":"object","properties":{}}`) + } + out = append(out, anthropicTool{Name: t.Function.Name, Description: t.Function.Description, InputSchema: schema}) + } + return out +} + +// chatStreamAnthropic is the streaming twin of chatAnthropic. Same retry +// doctrine as ChatStream: httpx.DoRetryOnce covers connection setup only — +// nothing is replayed once response bytes arrived. +func chatStreamAnthropic(ctx context.Context, b backends.Backend, model string, + msgs []ChatMsg, tools []ToolDef, opts Options, timeout time.Duration, + onEvent func(StreamEvent) error) (*StreamResult, error) { + system, messages := anthropicMessages(msgs) + req := anthropicRequest{ + Model: model, + MaxTokens: anthropicMaxTokens(opts), + System: anthropicSystem(system), + Messages: messages, + Stream: true, + OutputConfig: anthropicOutputConfig{Effort: resolveAnthropicEffort(opts, &b)}, + Tools: anthropicTools(tools), + // Top-level automatic caching: the API places a breakpoint on the + // last cacheable block, so each iteration of the tool loop (and + // each turn of the session) reads the growing history from cache + // instead of paying full input price for it; the explicit system + // breakpoint stays the stable-prefix anchor. + CacheControl: &anthropicCacheControl{Type: "ephemeral"}, + } + body, err := buildAnthropicBody(req, opts, &b) + if err != nil { + return nil, fmt.Errorf("llm: marshal request: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, b.Host+"/v1/messages", bytes.NewReader(body)) //nolint:gosec // G704: host from the F3 backend pool (context_backends) — admin-gated + locality-validated, no free user input. + if err != nil { + return nil, fmt.Errorf("llm: create request: %w", err) + } + setAnthropicHeaders(httpReq, &b) + httpReq.Header.Set("Accept", "text/event-stream") + + resp, err := httpx.DoRetryOnce(httpClient, httpReq, body) + if err != nil { + return nil, fmt.Errorf("llm: request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + // Typed StatusError for the failover classifier, as on every wire. + return nil, anthropicStatusError(resp) + } + + return parseAnthropicSSE(resp.Body, onEvent) +} + +// --- SSE parsing ---. + +// anthropicStreamFrame is the union of every data frame; only the fields +// the parser reads are modelled. +type anthropicStreamFrame struct { + Type string `json:"type"` + Index int `json:"index"` + Message *struct { + ID string `json:"id"` + Usage anthropicUsage `json:"usage"` + } `json:"message"` + ContentBlock *struct { + Type string `json:"type"` + ID string `json:"id"` + Name string `json:"name"` + Data string `json:"data"` // redacted_thinking payload + } `json:"content_block"` + Delta *struct { + Type string `json:"type"` + Text string `json:"text"` + PartialJSON string `json:"partial_json"` + Thinking string `json:"thinking"` + Signature string `json:"signature"` + StopReason string `json:"stop_reason"` + } `json:"delta"` + Usage *anthropicUsage `json:"usage"` + Error *struct { + Type string `json:"type"` + Message string `json:"message"` + } `json:"error"` +} + +// pendingThinking accumulates one thinking block across its deltas. +type pendingThinking struct { + text strings.Builder + signature string +} + +// anthropicStreamState carries the assembly across frames. +type anthropicStreamState struct { + onEvent func(StreamEvent) error + content strings.Builder + calls []*pendingToolCall + byIndex map[int]*pendingToolCall + thinking map[int]*pendingThinking + blocks []json.RawMessage // reasoning blocks, stream order, replayed verbatim + finish string + usage anthropicUsage + stopSeen bool + done bool + messageID string +} + +func (st *anthropicStreamState) emit(ev StreamEvent) error { + if st.onEvent == nil { + return nil + } + if err := st.onEvent(ev); err != nil { + return fmt.Errorf("llm: stream event callback: %w", err) + } + return nil +} + +// parseAnthropicSSE reads frames until message_stop or EOF. Lenient ending +// mirrors parseSSEStream: EOF after message_delta (stop reason known) is a +// complete turn; EOF before it is a hard error — a mid-stream connection +// death must not look like a finished answer. +func parseAnthropicSSE(r io.Reader, onEvent func(StreamEvent) error) (*StreamResult, error) { + st := &anthropicStreamState{onEvent: onEvent, byIndex: map[int]*pendingToolCall{}, thinking: map[int]*pendingThinking{}} + br := bufio.NewReader(r) + var data []string + for { + line, readErr := br.ReadString('\n') + if err := st.feedLine(strings.TrimRight(line, "\r\n"), &data); err != nil { + return nil, err + } + if st.done { + break + } + if readErr != nil { + if errors.Is(readErr, io.EOF) { + if err := st.dispatchData(&data); err != nil { + return nil, err + } + break + } + return nil, fmt.Errorf("llm: read stream: %w", readErr) + } + } + if !st.done && !st.stopSeen { + return nil, fmt.Errorf("llm: stream ended unexpectedly (no message_stop, no message_delta stop_reason)") + } + return st.result(), nil +} + +// feedLine: blank → dispatch the buffered frame; ":" → comment; "data:" → +// buffer; "event:"/"id:"/"retry:" → ignored (the type travels in the data). +func (st *anthropicStreamState) feedLine(line string, data *[]string) error { + switch { + case line == "": + return st.dispatchData(data) + case strings.HasPrefix(line, ":"): + return nil + case strings.HasPrefix(line, "data:"): + v := strings.TrimPrefix(line, "data:") + v = strings.TrimPrefix(v, " ") + *data = append(*data, v) + return nil + default: + return nil + } +} + +func (st *anthropicStreamState) dispatchData(data *[]string) error { + if len(*data) == 0 { + return nil + } + payload := strings.Join(*data, "\n") + *data = (*data)[:0] + return st.handleFrame(payload) +} + +func (st *anthropicStreamState) handleFrame(payload string) error { + var f anthropicStreamFrame + if err := json.Unmarshal([]byte(payload), &f); err != nil { + return fmt.Errorf("llm: malformed stream chunk: %w", err) + } + switch f.Type { + case "error": + if f.Error == nil { + return &StreamError{Message: "stream error event without an error object"} + } + return &StreamError{Code: f.Error.Type, Message: f.Error.Message} + case "message_start": + if f.Message != nil { + st.messageID = f.Message.ID + st.usage = f.Message.Usage + } + case "content_block_start": + return st.onBlockStart(&f) + case "content_block_delta": + return st.onBlockDelta(&f) + case "content_block_stop": + st.onBlockStop(f.Index) + case "message_delta": + return st.onMessageDelta(&f) + case "message_stop": + st.done = true + } + // ping and unknown frame types: ignored. + return nil +} + +// onBlockStart opens a content block. The API puts a tool_use block's id +// and name on the start frame (never on a later delta, unlike the OpenAI +// index-keyed assembly). A nameless tool_use block — a proxy artefact, not +// an API shape — is dropped rather than persisted as an empty call the +// harness would dispatch and replay as tool_use{id:""} (a 400 on the next +// turn); its input_json_delta frames then fall on no pending call. +func (st *anthropicStreamState) onBlockStart(f *anthropicStreamFrame) error { + if f.ContentBlock == nil { + return nil + } + switch f.ContentBlock.Type { + case "tool_use": + if f.ContentBlock.Name == "" { + return nil + } + pc := &pendingToolCall{index: f.Index, id: f.ContentBlock.ID, typ: "function", name: f.ContentBlock.Name, nameEmitted: true} + st.byIndex[f.Index] = pc + st.calls = append(st.calls, pc) + return st.emit(StreamEvent{ToolCallName: pc.name}) + case "thinking": + st.thinking[f.Index] = &pendingThinking{} + case "redacted_thinking": + // Arrives complete on the block start; replayed as-is. + raw, _ := json.Marshal(anthropicRedactedThinkingBlock{Type: "redacted_thinking", Data: f.ContentBlock.Data}) + st.blocks = append(st.blocks, raw) + } + return nil +} + +// onBlockDelta routes a delta by its own type: text to the sink, tool +// input to the pending call, reasoning to the pending thinking block +// (accumulated for the replay only — never surfaced; empty under the +// default display anyway). +func (st *anthropicStreamState) onBlockDelta(f *anthropicStreamFrame) error { + if f.Delta == nil { + return nil + } + switch f.Delta.Type { + case "text_delta": + if f.Delta.Text != "" { + st.content.WriteString(f.Delta.Text) + return st.emit(StreamEvent{ContentDelta: f.Delta.Text}) + } + case "input_json_delta": + if pc := st.byIndex[f.Index]; pc != nil { + pc.args.WriteString(f.Delta.PartialJSON) + } + case "thinking_delta": + if pt := st.thinking[f.Index]; pt != nil { + pt.text.WriteString(f.Delta.Thinking) + } + case "signature_delta": + if pt := st.thinking[f.Index]; pt != nil { + pt.signature = f.Delta.Signature + } + } + return nil +} + +// onBlockStop seals a thinking block into its replay form. +func (st *anthropicStreamState) onBlockStop(index int) { + pt := st.thinking[index] + if pt == nil { + return + } + raw, _ := json.Marshal(anthropicThinkingBlock{Type: "thinking", Thinking: pt.text.String(), Signature: pt.signature}) + st.blocks = append(st.blocks, raw) + delete(st.thinking, index) +} + +// onMessageDelta takes the stop reason and the output count. Newer servers +// repeat the (cumulative) input counters here; each one is taken only when +// it is itself present (> 0), so a delta that carries input_tokens alone +// cannot zero the cache counters message_start reported. +func (st *anthropicStreamState) onMessageDelta(f *anthropicStreamFrame) error { + if f.Usage != nil { + st.usage.OutputTokens = f.Usage.OutputTokens + if f.Usage.InputTokens > 0 { + st.usage.InputTokens = f.Usage.InputTokens + } + if f.Usage.CacheReadInputTokens > 0 { + st.usage.CacheReadInputTokens = f.Usage.CacheReadInputTokens + } + if f.Usage.CacheCreationInputTokens > 0 { + st.usage.CacheCreationInputTokens = f.Usage.CacheCreationInputTokens + } + } + if f.Delta != nil && f.Delta.StopReason != "" && st.finish == "" { + st.stopSeen = true + st.finish = mapAnthropicStopReason(f.Delta.StopReason) + return st.emit(StreamEvent{FinishReason: st.finish}) + } + return nil +} + +func (st *anthropicStreamState) result() *StreamResult { + prompt, output, cacheRead, cacheCreate := anthropicUsageTotals(st.usage) + res := &StreamResult{ + Content: st.content.String(), + FinishReason: st.finish, + PromptTokens: prompt, + CompletionTokens: output, + CacheReadTokens: cacheRead, + CacheCreationTokens: cacheCreate, + ProviderRequestID: st.messageID, + ProviderBlocks: st.blocks, + } + for _, pc := range st.calls { + args := bytes.TrimSpace(pc.args.Bytes()) + if len(args) == 0 { + args = []byte("{}") + } + res.ToolCalls = append(res.ToolCalls, ToolCall{ + ID: pc.id, + Type: "function", + Function: ToolCallFunction{Name: pc.name, Arguments: args}, + }) + } + return res +} diff --git a/go/internal/llm/anthropic_stream_test.go b/go/internal/llm/anthropic_stream_test.go new file mode 100644 index 00000000..f2efd645 --- /dev/null +++ b/go/internal/llm/anthropic_stream_test.go @@ -0,0 +1,635 @@ +// Native Anthropic Messages API wire (anthropic_stream.go), streaming path: +// fixture parsing (text, thinking, tool use, mid-stream error), lenient +// and hard endings, HTTP status errors, the request shape of a multi-turn +// tool round trip, and the ChatStreamBackend dispatch. Fixtures are +// hand-written from the Messages API event reference (claude-api, 2026-10), +// not live captures — what they cannot prove is live-only (cache hits, +// effort effect, 529 behaviour). +package llm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/httpx" +) + +func anthropicStreamBackend(host string) backends.Backend { + b := anthropicTestBackend(host) + b.ModelMap = map[string]backends.ModelSpec{"default": {Model: "claude-opus-5-5"}} + return b +} + +func TestAnthropicStreamContentFixture(t *testing.T) { + srv := serveFixture(t, "anthropic_content.sse") + defer srv.Close() + var events []StreamEvent + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "claude-opus-5-5", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 64}, 5*time.Second, collectEvents(&events)) + if err != nil { + t.Fatalf("ChatStreamBackend: %v", err) + } + if res.Content != "Berlin is sunny." { + t.Errorf("content = %q (thinking deltas must never leak)", res.Content) + } + if res.FinishReason != "stop" { + t.Errorf("finish = %q, want end_turn mapped to stop", res.FinishReason) + } + if res.PromptTokens != 312 || res.CompletionTokens != 9 { + t.Errorf("tokens = %d/%d, want 12+300 prompt (total incl. cache read) and 9 output", res.PromptTokens, res.CompletionTokens) + } + if res.CacheReadTokens != 300 || res.CacheCreationTokens != 0 { + t.Errorf("cache counters = %d/%d, want 300/0", res.CacheReadTokens, res.CacheCreationTokens) + } + if res.ProviderRequestID != "msg_01stream" { + t.Errorf("ProviderRequestID = %q", res.ProviderRequestID) + } + if len(res.ToolCalls) != 0 { + t.Errorf("tool calls = %v, want none", res.ToolCalls) + } + var joined strings.Builder + finishes := 0 + for i, ev := range events { + joined.WriteString(ev.ContentDelta) + if ev.ToolCallName != "" { + t.Errorf("unexpected tool event %+v", ev) + } + if ev.FinishReason != "" { + finishes++ + if i != len(events)-1 { + t.Errorf("finish event at %d of %d, want last", i, len(events)) + } + } + } + if joined.String() != res.Content || finishes != 1 { + t.Errorf("deltas = %q finishes = %d", joined.String(), finishes) + } +} + +func TestAnthropicStreamToolUseFixture(t *testing.T) { + srv := serveFixture(t, "anthropic_tool_use.sse") + defer srv.Close() + var events []StreamEvent + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "claude-opus-5-5", + []ChatMsg{{Role: "user", Content: "weather?"}}, nil, Options{NumPredict: 64}, 5*time.Second, collectEvents(&events)) + if err != nil { + t.Fatalf("ChatStreamBackend: %v", err) + } + if res.FinishReason != "tool_calls" { + t.Errorf("finish = %q, want tool_use mapped to tool_calls (the harness's dispatch key)", res.FinishReason) + } + if res.Content != "Let me check." { + t.Errorf("content = %q", res.Content) + } + if len(res.ToolCalls) != 2 { + t.Fatalf("tool calls = %d, want 2", len(res.ToolCalls)) + } + first := res.ToolCalls[0] + if first.ID != "toolu_01weather" || first.Type != "function" || first.Function.Name != "get_weather" { + t.Errorf("first call = %+v", first) + } + args := argsAsMap(t, first) + if args["city"] != "Berlin" || args["unit"] != "c" { + t.Errorf("arguments = %v, want the partial_json fragments joined into one object", args) + } + second := res.ToolCalls[1] + if second.ID != "toolu_02time" || second.Function.Name != "get_time" || string(second.Function.Arguments) != "{}" { + t.Errorf("second call = %+v, want empty arguments as {}", second) + } + if res.PromptTokens != 552 || res.CompletionTokens != 31 || res.CacheCreationTokens != 512 { + t.Errorf("tokens = %d/%d cache_create=%d, want 40+512 / 31 / 512", res.PromptTokens, res.CompletionTokens, res.CacheCreationTokens) + } + var names []string + for _, ev := range events { + if ev.ToolCallName != "" { + names = append(names, ev.ToolCallName) + } + } + if strings.Join(names, ",") != "get_weather,get_time" { + t.Errorf("tool name events = %v, want one per call in stream order", names) + } + if last := events[len(events)-1]; last.FinishReason != "tool_calls" { + t.Errorf("last event = %+v, want the finish", last) + } + + // The assembled calls round-trip into the next request as tool_use + // blocks with the SAME ids, so tool_result blocks can reference them. + system, msgs := anthropicMessages([]ChatMsg{ + {Role: "system", Content: "sys"}, + {Role: "user", Content: "weather?"}, + {Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls}, + {Role: "tool", Content: "21C", ToolCallID: "toolu_01weather"}, + {Role: "tool", Content: "12:00", ToolCallID: "toolu_02time"}, + }) + if system != "sys" || len(msgs) != 3 { + t.Fatalf("system=%q msgs=%d, want 3 turns (user, assistant, merged tool results)", system, len(msgs)) + } + raw, _ := json.Marshal(msgs) + for _, want := range []string{ + `"role":"assistant"`, `"type":"tool_use","id":"toolu_01weather","name":"get_weather","input":{"city":"Berlin","unit":"c"}`, + `"type":"tool_use","id":"toolu_02time","name":"get_time","input":{}`, + `"type":"tool_result","tool_use_id":"toolu_01weather","content":"21C"`, + `"type":"tool_result","tool_use_id":"toolu_02time","content":"12:00"`, + } { + if !strings.Contains(string(raw), want) { + t.Errorf("messages lack %s:\n%s", want, raw) + } + } + if msgs[2].Role != "user" { + t.Errorf("tool results must travel as a user turn, got %q", msgs[2].Role) + } +} + +func TestAnthropicStreamMidStreamError(t *testing.T) { + srv := serveFixture(t, "anthropic_midstream_error.sse") + defer srv.Close() + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + var se *StreamError + if !errors.As(err, &se) { + t.Fatalf("want *StreamError, got %v (res %v)", err, res) + } + if se.Code != "overloaded_error" || se.Message != "Overloaded" { + t.Errorf("StreamError = %+v", se) + } + if res != nil { + t.Errorf("result must be nil on a mid-stream error, got %+v", res) + } +} + +func TestAnthropicStreamEOFWithoutStopIsError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":1}}}\n\n" + + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"half\"}}\n\n")) + })) + defer srv.Close() + _, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + if err == nil || !strings.Contains(err.Error(), "ended unexpectedly") { + t.Errorf("err = %v, want the unexpected-end error (a dead connection must not look like a finished turn)", err) + } +} + +func TestAnthropicStreamEOFAfterMessageDeltaAccepted(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":3}}}\n\n" + + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"done\"}}\n\n" + + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"max_tokens\"},\"usage\":{\"output_tokens\":4}}")) + })) + defer srv.Close() + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + if err != nil { + t.Fatalf("EOF after message_delta must be accepted: %v", err) + } + if res.Content != "done" || res.FinishReason != "length" || res.PromptTokens != 3 || res.CompletionTokens != 4 { + t.Errorf("result = %+v", res) + } +} + +func TestAnthropicStreamHTTPErrorStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(529) + _, _ = w.Write([]byte(`{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}`)) + })) + defer srv.Close() + _, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + var se *httpx.StatusError + if !errors.As(err, &se) || se.Code != 529 { + t.Fatalf("want *httpx.StatusError 529, got %v", err) + } + if got := backends.Classify(err, backends.ProviderGeneric); got != backends.ClassGateway { + t.Errorf("Classify = %v, want Gateway (529 overloaded fails over)", got) + } +} + +func TestAnthropicStreamCallbackErrorAborts(t *testing.T) { + srv := serveFixture(t, "anthropic_content.sse") + defer srv.Close() + wantErr := errors.New("sink closed") + calls := 0 + _, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, + func(StreamEvent) error { calls++; return wantErr }) + if !errors.Is(err, wantErr) || calls != 1 { + t.Errorf("err = %v calls = %d, want the callback error after one event", err, calls) + } +} + +// TestAnthropicStreamRequestWire pins the multi-turn tool round trip on the +// wire: headers, stream flag, cached system block, user/assistant/tool +// mapping with consecutive tool results folded into ONE user turn, tool +// definitions as input_schema, effort, and the absence of every field this +// wire must not carry. +func TestAnthropicStreamRequestWire(t *testing.T) { + srv, body, hdr := captureRequestBody(t, "data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":1}}}\n\n"+ + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":1}}\n\n"+ + "data: {\"type\":\"message_stop\"}\n\n") + defer srv.Close() + + tools := []ToolDef{ + {Type: "function", Function: ToolDefFunction{ + Name: "get_weather", Description: "Weather for a city", + Parameters: json.RawMessage(`{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}`), + }}, + {Type: "function", Function: ToolDefFunction{Name: "get_time"}}, + } + msgs := []ChatMsg{ + {Role: "system", Content: "You are a weather demo assistant."}, + {Role: "user", Content: "Weather in Berlin?"}, + {Role: "assistant", Content: "Checking.", ToolCalls: []ToolCall{ + {ID: "toolu_1", Type: "function", Function: ToolCallFunction{Name: "get_weather", Arguments: json.RawMessage(`{"city":"Berlin"}`)}}, + {ID: "toolu_2", Type: "function", Function: ToolCallFunction{Name: "get_time", Arguments: json.RawMessage(`not json`)}}, + }}, + {Role: "tool", Content: `{"temperature_c":21}`, ToolCallID: "toolu_1"}, + {Role: "tool", Content: `12:00`, ToolCallID: "toolu_2"}, + } + b := anthropicStreamBackend(srv.URL) + b.ExtraBody = map[string]any{"effort": "high"} + opts := Options{Temperature: 0.7, RepeatPenalty: 1.1, NumPredict: 256} + if _, err := ChatStreamBackend(context.Background(), b, "claude-opus-5-5", msgs, tools, opts, 5*time.Second, nil); err != nil { + t.Fatalf("ChatStreamBackend: %v", err) + } + + if got := hdr.Get("x-api-key"); got != "sk-ant-test" { + t.Errorf("x-api-key = %q", got) + } + if got := hdr.Get("anthropic-version"); got != anthropicVersion { + t.Errorf("anthropic-version = %q", got) + } + if got := hdr.Get("Accept"); got != "text/event-stream" { + t.Errorf("Accept = %q", got) + } + if hdr.Get("Authorization") != "" { + t.Error("Authorization header sent on the anthropic wire") + } + + var req map[string]any + if err := json.Unmarshal(*body, &req); err != nil { + t.Fatalf("request body not JSON: %v\n%s", err, *body) + } + if req["stream"] != true || req["model"] != "claude-opus-5-5" || req["max_tokens"] != float64(256) { + t.Errorf("stream/model/max_tokens = %v/%v/%v", req["stream"], req["model"], req["max_tokens"]) + } + if oc, _ := req["output_config"].(map[string]any); oc["effort"] != "high" { + t.Errorf("output_config = %v, want extra_body effort high", req["output_config"]) + } + for _, k := range anthropicForbiddenKeys { + if v, has := req[k]; has { + t.Errorf("body carries %q = %v", k, v) + } + } + sys, _ := req["system"].([]any) + if len(sys) != 1 { + t.Fatalf("system = %v", req["system"]) + } + if blk, _ := sys[0].(map[string]any); blk["text"] != "You are a weather demo assistant." || blk["cache_control"] == nil { + t.Errorf("system block = %v, want the lifted system row with cache_control", blk) + } + + turns, _ := req["messages"].([]any) + if len(turns) != 3 { + t.Fatalf("messages = %d turns, want user / assistant / merged tool results:\n%s", len(turns), *body) + } + role := func(i int) string { m, _ := turns[i].(map[string]any); r, _ := m["role"].(string); return r } + blocks := func(i int) []any { m, _ := turns[i].(map[string]any); c, _ := m["content"].([]any); return c } + if role(0) != "user" || role(1) != "assistant" || role(2) != "user" { + t.Errorf("roles = %s/%s/%s", role(0), role(1), role(2)) + } + if ub := blocks(0); len(ub) != 1 || ub[0].(map[string]any)["text"] != "Weather in Berlin?" { + t.Errorf("user turn = %v", ub) + } + ab := blocks(1) + if len(ab) != 3 { + t.Fatalf("assistant turn = %v, want text + 2 tool_use", ab) + } + if tu, _ := ab[1].(map[string]any); tu["type"] != "tool_use" || tu["id"] != "toolu_1" || tu["name"] != "get_weather" { + t.Errorf("tool_use[0] = %v", tu) + } else if in, _ := tu["input"].(map[string]any); in["city"] != "Berlin" { + t.Errorf("tool_use[0].input = %v", tu["input"]) + } + if tu, _ := ab[2].(map[string]any); tu["id"] != "toolu_2" { + t.Errorf("tool_use[1] = %v", tu) + } else if in, _ := tu["input"].(map[string]any); in == nil || len(in) != 0 { + t.Errorf("tool_use[1].input = %v, want {} for unparseable arguments", tu["input"]) + } + tr := blocks(2) + if len(tr) != 2 { + t.Fatalf("tool-result turn = %v, want both results folded into one user turn", tr) + } + if r0, _ := tr[0].(map[string]any); r0["type"] != "tool_result" || r0["tool_use_id"] != "toolu_1" || r0["content"] != `{"temperature_c":21}` { + t.Errorf("tool_result[0] = %v", r0) + } + if r1, _ := tr[1].(map[string]any); r1["tool_use_id"] != "toolu_2" || r1["content"] != "12:00" { + t.Errorf("tool_result[1] = %v", r1) + } + for i := range turns { + if _, has := turns[i].(map[string]any)["tool_calls"]; has { + t.Errorf("turn %d carries the OpenAI tool_calls field", i) + } + } + + wt, _ := req["tools"].([]any) + if len(wt) != 2 { + t.Fatalf("tools = %v", req["tools"]) + } + if t0, _ := wt[0].(map[string]any); t0["name"] != "get_weather" || t0["description"] != "Weather for a city" || t0["input_schema"] == nil || t0["function"] != nil { + t.Errorf("tools[0] = %v, want the Messages API shape (name/description/input_schema)", t0) + } + if t1, _ := wt[1].(map[string]any); t1["input_schema"] == nil { + t.Errorf("tools[1] = %v, want an empty object schema for a parameterless tool", t1) + } +} + +// TestChatStreamBackendDispatch: the protocol field, not the host, picks +// the wire — an openai row keeps /v1/chat/completions with Bearer auth +// and its extra_body, an anthropic row takes /v1/messages. +func TestChatStreamBackendDispatch(t *testing.T) { + type hit struct{ path, auth, key string } + var hits []hit + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits = append(hits, hit{r.URL.Path, r.Header.Get("Authorization"), r.Header.Get("x-api-key")}) + if r.URL.Path == "/v1/messages" { + _, _ = w.Write([]byte("data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\ndata: {\"type\":\"message_stop\"}\n\n")) + return + } + _, _ = w.Write([]byte(minimalStopStream)) + })) + defer srv.Close() + msgs := []ChatMsg{{Role: "user", Content: "hi"}} + + oa := backends.Backend{Host: srv.URL, Protocol: backends.ProtocolOpenAI, APIKey: "sk-oa"} + if _, err := ChatStreamBackend(context.Background(), oa, "m", msgs, nil, Options{NumPredict: 4}, 5*time.Second, nil); err != nil { + t.Fatalf("openai: %v", err) + } + ol := backends.Backend{Host: srv.URL, Protocol: backends.ProtocolOllama} + if _, err := ChatStreamBackend(context.Background(), ol, "m", msgs, nil, Options{NumPredict: 4}, 5*time.Second, nil); err != nil { + t.Fatalf("ollama: %v", err) + } + if _, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", msgs, nil, Options{NumPredict: 4}, 5*time.Second, nil); err != nil { + t.Fatalf("anthropic: %v", err) + } + want := []hit{ + {"/v1/chat/completions", "Bearer sk-oa", ""}, + {"/v1/chat/completions", "", ""}, + {"/v1/messages", "", "sk-ant-test"}, + } + if len(hits) != len(want) { + t.Fatalf("hits = %v", hits) + } + for i := range want { + if hits[i] != want[i] { + t.Errorf("hit %d = %+v, want %+v", i, hits[i], want[i]) + } + } +} + +// TestAnthropicMessagesEdgeCases: consecutive user turns fold, empty +// assistant turns drop, several system rows join. +func TestAnthropicMessagesEdgeCases(t *testing.T) { + system, msgs := anthropicMessages([]ChatMsg{ + {Role: "system", Content: "a"}, + {Role: "system", Content: "b"}, + {Role: "user", Content: "one"}, + {Role: "user", Content: "two"}, + {Role: "assistant", Content: ""}, + {Role: "user", Content: "three"}, + }) + if system != "a\n\nb" { + t.Errorf("system = %q", system) + } + if len(msgs) != 1 || msgs[0].Role != "user" { + t.Fatalf("msgs = %+v, want one folded user turn (empty assistant dropped)", msgs) + } + if blocks, _ := msgs[0].Content.([]any); len(blocks) != 3 { + t.Errorf("folded user turn = %v, want three text blocks", msgs[0].Content) + } +} + +// --- review follow-ups (adversarial pass, 2026-10-11) ---. + +// TestAnthropicStreamMessageDeltaUsageKeepsCacheCounters: a message_delta +// that repeats input_tokens but omits the cache fields (older servers, +// proxies) must not zero the counters message_start reported. +func TestAnthropicStreamMessageDeltaUsageKeepsCacheCounters(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":10,\"cache_read_input_tokens\":400,\"cache_creation_input_tokens\":50}}}\n\n" + + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"input_tokens\":12,\"output_tokens\":4}}\n\n" + + "data: {\"type\":\"message_stop\"}\n\n")) + })) + defer srv.Close() + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + if err != nil { + t.Fatal(err) + } + if res.CacheReadTokens != 400 || res.CacheCreationTokens != 50 || res.PromptTokens != 12+400+50 || res.CompletionTokens != 4 { + t.Errorf("usage = prompt %d cache %d/%d out %d, want 462 / 400/50 / 4", res.PromptTokens, res.CacheReadTokens, res.CacheCreationTokens, res.CompletionTokens) + } +} + +// TestAnthropicStreamNamelessToolUseDropped: a tool_use block start without +// a name (proxy artefact) yields no call and no event; its input deltas +// fall on no pending call. +func TestAnthropicStreamNamelessToolUseDropped(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":1}}}\n\n" + + "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"\",\"name\":\"\",\"input\":{}}}\n\n" + + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{}\"}}\n\n" + + "data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_ok\",\"name\":\"get_time\",\"input\":{}}}\n\n" + + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\"},\"usage\":{\"output_tokens\":2}}\n\n" + + "data: {\"type\":\"message_stop\"}\n\n")) + })) + defer srv.Close() + var events []StreamEvent + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, collectEvents(&events)) + if err != nil { + t.Fatal(err) + } + if len(res.ToolCalls) != 1 || res.ToolCalls[0].ID != "toolu_ok" { + t.Errorf("tool calls = %+v, want only the named one", res.ToolCalls) + } + names := 0 + for _, ev := range events { + if ev.ToolCallName != "" { + names++ + } + } + if names != 1 { + t.Errorf("tool name events = %d, want 1", names) + } +} + +// TestAnthropicMessagesRepairsEmptyToolIDs: a history served earlier by an +// openai-wire row (llama.cpp streams calls without ids) replays with +// synthetic, matching ids instead of a 400 on tool_use.id. +func TestAnthropicMessagesRepairsEmptyToolIDs(t *testing.T) { + _, msgs := anthropicMessages([]ChatMsg{ + {Role: "user", Content: "q"}, + {Role: "assistant", ToolCalls: []ToolCall{ + {Function: ToolCallFunction{Name: "a", Arguments: json.RawMessage(`{}`)}}, + {ID: "real", Function: ToolCallFunction{Name: "b", Arguments: json.RawMessage(`{}`)}}, + {Function: ToolCallFunction{Name: "c", Arguments: json.RawMessage(`{}`)}}, + }}, + {Role: "tool", Content: "ra"}, + {Role: "tool", Content: "rb", ToolCallID: "real"}, + {Role: "tool", Content: "rc"}, + }) + raw, _ := json.Marshal(msgs) + s := string(raw) + for _, want := range []string{ + `"type":"tool_use","id":"ctxcall_1","name":"a"`, `"type":"tool_use","id":"real","name":"b"`, `"type":"tool_use","id":"ctxcall_2","name":"c"`, + `"tool_use_id":"ctxcall_1","content":"ra"`, `"tool_use_id":"real","content":"rb"`, `"tool_use_id":"ctxcall_2","content":"rc"`, + } { + if !strings.Contains(s, want) { + t.Errorf("messages lack %s:\n%s", want, s) + } + } + if strings.Contains(s, `"id":""`) || strings.Contains(s, `"tool_use_id":""`) { + t.Errorf("an empty id reached the wire:\n%s", s) + } +} + +// TestAnthropicStreamThinkingBlocksReplayFirst: the fixture's thinking block +// (empty text, signature — the default display) is re-assembled into +// StreamResult.ProviderBlocks and placed FIRST in the replayed assistant +// turn, ahead of text and tool_use — the order the API produced them in, +// which the reference asks to be kept while a tool_use awaits its result. +func TestAnthropicStreamThinkingBlocksReplayFirst(t *testing.T) { + srv := serveFixture(t, "anthropic_tool_use.sse") + defer srv.Close() + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "weather?"}}, nil, Options{NumPredict: 64}, 5*time.Second, nil) + if err != nil { + t.Fatal(err) + } + if len(res.ProviderBlocks) != 1 { + t.Fatalf("ProviderBlocks = %d, want the one thinking block", len(res.ProviderBlocks)) + } + if got := string(res.ProviderBlocks[0]); got != `{"type":"thinking","thinking":"","signature":"sigABC"}` { + t.Errorf("thinking block = %s", got) + } + if res.Content != "Let me check." || len(res.ToolCalls) != 2 { + t.Errorf("content/calls disturbed: %q / %d", res.Content, len(res.ToolCalls)) + } + + _, msgs := anthropicMessages([]ChatMsg{ + {Role: "user", Content: "weather?"}, + {Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls, ProviderBlocks: res.ProviderBlocks}, + {Role: "tool", Content: "21C", ToolCallID: "toolu_01weather"}, + {Role: "tool", Content: "12:00", ToolCallID: "toolu_02time"}, + }) + blocks, _ := msgs[1].Content.([]any) + if len(blocks) != 4 { + t.Fatalf("assistant turn = %d blocks, want thinking + text + 2 tool_use", len(blocks)) + } + first, _ := json.Marshal(blocks[0]) + if !strings.HasPrefix(string(first), `{"type":"thinking"`) { + t.Errorf("first block = %s, want the thinking block", first) + } + + // The openai wire never sees the field. + raw, _ := json.Marshal(ChatMsg{Role: "assistant", Content: "x", ProviderBlocks: res.ProviderBlocks}) + if strings.Contains(string(raw), "thinking") { + t.Errorf("ProviderBlocks leaked into the OpenAI message object: %s", raw) + } +} + +// TestAnthropicStreamRedactedThinkingCarried: a redacted_thinking block +// arrives complete on its start frame and is replayed as-is. +func TestAnthropicStreamRedactedThinkingCarried(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":1}}}\n\n" + + "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"redacted_thinking\",\"data\":\"EmwKAhgB\"}}\n\n" + + "data: {\"type\":\"content_block_stop\",\"index\":0}\n\n" + + "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"ok\"}}\n\n" + + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":1}}\n\n" + + "data: {\"type\":\"message_stop\"}\n\n")) + })) + defer srv.Close() + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + if err != nil { + t.Fatal(err) + } + if len(res.ProviderBlocks) != 1 || string(res.ProviderBlocks[0]) != `{"type":"redacted_thinking","data":"EmwKAhgB"}` { + t.Errorf("ProviderBlocks = %v", res.ProviderBlocks) + } +} + +// TestAnthropicMessagesRepairsOrphanToolUse: a tool_use the history never +// answered (interrupted turn) gets a synthetic is_error tool_result before +// the next user text, and at the end of the history — otherwise every later +// turn of the session would 400 on this wire. +func TestAnthropicMessagesRepairsOrphanToolUse(t *testing.T) { + _, msgs := anthropicMessages([]ChatMsg{ + {Role: "user", Content: "q1"}, + {Role: "assistant", ToolCalls: []ToolCall{ + {ID: "A", Function: ToolCallFunction{Name: "a", Arguments: json.RawMessage(`{}`)}}, + {ID: "B", Function: ToolCallFunction{Name: "b", Arguments: json.RawMessage(`{}`)}}, + }}, + {Role: "tool", Content: "ra", ToolCallID: "A"}, + {Role: "user", Content: "q2"}, + {Role: "assistant", ToolCalls: []ToolCall{{ID: "C", Function: ToolCallFunction{Name: "c", Arguments: json.RawMessage(`{}`)}}}}, + }) + if len(msgs) != 5 { + t.Fatalf("turns = %d, want user / assistant / user[ra, B-orphan, q2] / assistant / user[C-orphan]", len(msgs)) + } + raw, _ := json.Marshal(msgs[2].Content) + s := string(raw) + if !strings.Contains(s, `"tool_use_id":"A","content":"ra"`) || + !strings.Contains(s, `"tool_use_id":"B","content":"`+orphanToolResult+`","is_error":true`) || + !strings.HasSuffix(s, `{"type":"text","text":"q2"}]`) { + t.Errorf("third turn = %s", s) + } + last, _ := json.Marshal(msgs[4].Content) + if msgs[4].Role != "user" || !strings.Contains(string(last), `"tool_use_id":"C"`) || !strings.Contains(string(last), `"is_error":true`) { + t.Errorf("trailing orphan not repaired: %s %s", msgs[4].Role, last) + } + // An answered call gets no synthetic result. + if strings.Count(s, `"tool_use_id":"A"`) != 1 { + t.Errorf("answered call duplicated: %s", s) + } +} + +// TestAnthropicStreamTopLevelCacheControl: the multi-turn stream path asks +// for the automatic breakpoint on the conversation tail. +func TestAnthropicStreamTopLevelCacheControl(t *testing.T) { + srv, body, _ := captureRequestBody(t, "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\ndata: {\"type\":\"message_stop\"}\n\n") + defer srv.Close() + if _, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil); err != nil { + t.Fatal(err) + } + var req map[string]any + _ = json.Unmarshal(*body, &req) + if cc, _ := req["cache_control"].(map[string]any); cc["type"] != "ephemeral" { + t.Errorf("top-level cache_control = %v, want {type: ephemeral}", req["cache_control"]) + } +} + +// TestStreamErrorClassifiesLikeItsStatus: the anthropic error types inside an +// HTTP-200 stream map onto the classes the equivalent status codes get. +func TestStreamErrorClassifiesLikeItsStatus(t *testing.T) { + cases := map[string]backends.ErrClass{ + "overloaded_error": backends.ClassGateway, "rate_limit_error": backends.ClassRateLimit, + "authentication_error": backends.ClassAuth, "invalid_request_error": backends.ClassBadRequest, + "api_error": backends.ClassServerFault, "502": backends.ClassServerFault, "": backends.ClassServerFault, + } + for code, want := range cases { + err := fmt.Errorf("wrapped: %w", &StreamError{Code: code, Message: "m"}) + if got := backends.Classify(err, backends.ProviderGeneric); got != want { + t.Errorf("Classify(StreamError %q) = %v, want %v", code, got, want) + } + } +} diff --git a/go/internal/llm/anthropic_wire_test.go b/go/internal/llm/anthropic_wire_test.go new file mode 100644 index 00000000..088b1810 --- /dev/null +++ b/go/internal/llm/anthropic_wire_test.go @@ -0,0 +1,433 @@ +// Native Anthropic Messages API wire (anthropic.go), non-streaming path: +// request shape, headers, effort resolution, extra_body merge, response +// decode (text/thinking blocks, usage incl. cache counters, stop-reason +// mapping, message id) and the error → failover-class mapping. Every probe +// runs against an httptest fake of api.anthropic.com — no live key in CI. +// What this file CANNOT prove (live only): an actual cache hit, the effort +// level's effect on output_tokens, 529 behaviour under load. +package llm + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/httpx" + "github.com/GottZ/ctx/internal/llmlog" + "github.com/GottZ/ctx/internal/promptguard" +) + +// anthropicRecorder captures path, decoded body and headers of one +// /v1/messages call and answers with a fixed JSON (or a fixed status). +type anthropicRecorder struct { + mu sync.Mutex + path string + body map[string]any + header http.Header + srv *httptest.Server +} + +func newAnthropicRecorder(t *testing.T, status int, respJSON string, respHeader map[string]string) *anthropicRecorder { + t.Helper() + rec := &anthropicRecorder{} + rec.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var m map[string]any + _ = json.NewDecoder(r.Body).Decode(&m) + rec.mu.Lock() + rec.path = r.URL.Path + rec.body = m + rec.header = r.Header.Clone() + rec.mu.Unlock() + for k, v := range respHeader { + w.Header().Set(k, v) + } + w.WriteHeader(status) + _, _ = w.Write([]byte(respJSON)) + })) + t.Cleanup(rec.srv.Close) + return rec +} + +func (rec *anthropicRecorder) recorded() (string, map[string]any, http.Header) { + rec.mu.Lock() + defer rec.mu.Unlock() + return rec.path, rec.body, rec.header +} + +// anthropicRespOK is a complete Messages API answer: a thinking block (empty +// text, the default display), two text blocks, cache counters on usage. +const anthropicRespOK = `{"id":"msg_01abc","type":"message","role":"assistant","model":"claude-opus-5-5", + "content":[{"type":"thinking","thinking":"","signature":"sig"}, + {"type":"text","text":"Hel"},{"type":"text","text":"lo"}], + "stop_reason":"end_turn","stop_sequence":null, + "usage":{"input_tokens":10,"output_tokens":7, + "cache_read_input_tokens":400,"cache_creation_input_tokens":100}}` + +func anthropicTestBackend(host string) backends.Backend { + return backends.Backend{ + Host: host, Protocol: backends.ProtocolAnthropic, + ProviderClass: backends.ProviderGeneric, Model: "claude-opus-5-5", + Trust: backends.TrustNoCredentials, Locality: backends.LocalityExternal, + APIKey: "sk-ant-test", APIKeyRef: "anthropic-chat", NumCtx: 100000, + } +} + +// sampling keys the Claude 5 models answer 400 to, plus every openai-wire +// field that has no meaning on this wire. None may ever reach the body. +var anthropicForbiddenKeys = []string{ + "temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", + "repeat_penalty", "logprobs", "top_logprobs", "response_format", "reasoning", "seed", + "thinking", "tool_choice", "stream_options", "options", "num_ctx", "effort", +} + +func TestAnthropicRequestWire(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + b.Think = "false" // ignored on this wire — thinking cannot be disabled + b.ExtraHeaders = map[string]string{"X-Title": "ctx", "anthropic-version": "1999-01-01"} + opts := Options{Temperature: 0.1, TopP: 0.9, TopK: 40, RepeatPenalty: 1.1, NumPredict: 5, NumCtx: 100000, TopLogprobs: 20} + if _, err := Chat(context.Background(), b, "sys prompt", "user prompt", opts, 5*time.Second); err != nil { + t.Fatalf("Chat: %v", err) + } + path, body, hdr := rec.recorded() + + if path != "/v1/messages" { + t.Errorf("path = %q, want /v1/messages", path) + } + if got := hdr.Get("x-api-key"); got != "sk-ant-test" { + t.Errorf("x-api-key = %q, want the resolved api_key_ref", got) + } + if got := hdr.Get("anthropic-version"); got != anthropicVersion { + t.Errorf("anthropic-version = %q, want %q (core header must beat extra_headers)", got, anthropicVersion) + } + if got := hdr.Get("Authorization"); got != "" { + t.Errorf("Authorization = %q, want none (the Messages API authenticates via x-api-key)", got) + } + if got := hdr.Get("X-Title"); got != "ctx" { + t.Errorf("X-Title = %q, want extra_headers to reach the wire", got) + } + if got := hdr.Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type = %q", got) + } + + if body["model"] != "claude-opus-5-5" { + t.Errorf("model = %v", body["model"]) + } + if body["max_tokens"] != float64(5) { + t.Errorf("max_tokens = %v, want NumPredict 5", body["max_tokens"]) + } + if _, has := body["stream"]; has { + t.Errorf("stream key present on the non-streaming path: %v", body["stream"]) + } + sys, _ := body["system"].([]any) + if len(sys) != 1 { + t.Fatalf("system = %v, want one cached text block", body["system"]) + } + blk, _ := sys[0].(map[string]any) + if blk["type"] != "text" || blk["text"] != "sys prompt" { + t.Errorf("system block = %v", blk) + } + if cc, _ := blk["cache_control"].(map[string]any); cc["type"] != "ephemeral" { + t.Errorf("system cache_control = %v, want {type: ephemeral}", blk["cache_control"]) + } + msgs, _ := body["messages"].([]any) + if len(msgs) != 1 { + t.Fatalf("messages = %v, want exactly the user turn", body["messages"]) + } + if m, _ := msgs[0].(map[string]any); m["role"] != "user" || m["content"] != "user prompt" { + t.Errorf("messages[0] = %v", msgs[0]) + } + if oc, _ := body["output_config"].(map[string]any); oc["effort"] != backends.AnthropicDefaultEffort { + t.Errorf("output_config = %v, want effort %q by default", body["output_config"], backends.AnthropicDefaultEffort) + } + for _, k := range anthropicForbiddenKeys { + if v, has := body[k]; has { + t.Errorf("body carries %q = %v — the Claude 5 models reject sampling fields and nothing else here is a Messages API field", k, v) + } + } +} + +// TestAnthropicEffortResolution: model_map params (Options.Extra) beat the +// row's extra_body, which beats the default; the key itself never lands +// top-level. +func TestAnthropicEffortResolution(t *testing.T) { + effortOf := func(t *testing.T, b backends.Backend, opts Options) (string, map[string]any) { + t.Helper() + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b.Host = rec.srv.URL + if _, err := Chat(context.Background(), b, "s", "u", opts, 5*time.Second); err != nil { + t.Fatalf("Chat: %v", err) + } + _, body, _ := rec.recorded() + oc, _ := body["output_config"].(map[string]any) + e, _ := oc["effort"].(string) + return e, body + } + + b := anthropicTestBackend("") + b.ExtraBody = map[string]any{"effort": "high"} + if e, body := effortOf(t, b, Options{NumPredict: 5}); e != "high" { + t.Errorf("extra_body effort: got %q, want high", e) + } else if _, leaked := body["effort"]; leaked { + t.Error("effort merged top-level — it must be lifted into output_config only") + } + + if e, _ := effortOf(t, b, Options{NumPredict: 5, Extra: map[string]any{"effort": "low"}}); e != "low" { + t.Errorf("model_map effort must beat extra_body: got %q, want low", e) + } + + if e, _ := effortOf(t, anthropicTestBackend(""), Options{NumPredict: 5}); e != backends.AnthropicDefaultEffort { + t.Errorf("default effort: got %q, want %q", e, backends.AnthropicDefaultEffort) + } +} + +// TestAnthropicExtraBodyMerge: everything but effort merges top-level with +// the row winning over model_map params (openai-wire parity) — the +// operator's hatch for fields this wire does not model. +func TestAnthropicExtraBodyMerge(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + b.ExtraBody = map[string]any{ + "thinking": map[string]any{"type": "adaptive", "display": "summarized"}, + "metadata": map[string]any{"user_id": "row"}, + } + opts := Options{NumPredict: 5, Extra: map[string]any{"metadata": map[string]any{"user_id": "params"}, "stop_sequences": []any{"END"}}} + if _, err := Chat(context.Background(), b, "s", "u", opts, 5*time.Second); err != nil { + t.Fatalf("Chat: %v", err) + } + _, body, _ := rec.recorded() + if th, _ := body["thinking"].(map[string]any); th["display"] != "summarized" { + t.Errorf("extra_body.thinking did not reach the wire: %v", body["thinking"]) + } + if md, _ := body["metadata"].(map[string]any); md["user_id"] != "row" { + t.Errorf("extra_body must win over model_map params on collision: %v", body["metadata"]) + } + if ss, _ := body["stop_sequences"].([]any); len(ss) != 1 || ss[0] != "END" { + t.Errorf("model_map param stop_sequences did not reach the wire: %v", body["stop_sequences"]) + } + if body["max_tokens"] != float64(5) { + t.Errorf("max_tokens = %v, the merge must not disturb the modelled fields", body["max_tokens"]) + } +} + +// TestAnthropicJSONModeIsNoOp pins the documented contract: ChatJSON sends +// the same body as Chat — no response_format, no output_config.format (that +// needs a schema ctx does not have), and the request still succeeds. +func TestAnthropicJSONModeIsNoOp(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + resp, err := ChatJSON(context.Background(), b, "s", "u", Options{NumPredict: 5}, 5*time.Second) + if err != nil { + t.Fatalf("ChatJSON: %v", err) + } + if resp.Message.Content != "Hello" { + t.Errorf("content = %q", resp.Message.Content) + } + _, body, _ := rec.recorded() + if _, has := body["response_format"]; has { + t.Error("response_format sent on the anthropic wire") + } + if oc, _ := body["output_config"].(map[string]any); oc["format"] != nil { + t.Errorf("output_config.format = %v, want none (no schema available)", oc["format"]) + } +} + +// TestAnthropicMaxTokensRequired: max_tokens cannot be omitted on this wire, +// so a bare Options{} gets the documented fallback instead of a 400. +func TestAnthropicMaxTokensRequired(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + if _, err := Chat(context.Background(), b, "s", "u", Options{}, 5*time.Second); err != nil { + t.Fatalf("Chat: %v", err) + } + _, body, _ := rec.recorded() + if body["max_tokens"] != float64(anthropicDefaultMaxTokens) { + t.Errorf("max_tokens = %v, want the fallback %d", body["max_tokens"], anthropicDefaultMaxTokens) + } +} + +// TestAnthropicEmptySystemOmitted: the API rejects an empty text block, so +// an empty system prompt must yield no system field at all. +func TestAnthropicEmptySystemOmitted(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + if _, err := Chat(context.Background(), b, "", "u", Options{NumPredict: 5}, 5*time.Second); err != nil { + t.Fatalf("Chat: %v", err) + } + _, body, _ := rec.recorded() + if _, has := body["system"]; has { + t.Errorf("system = %v, want the field omitted for an empty prompt", body["system"]) + } +} + +// TestAnthropicResponseDecode: text blocks concatenate, thinking blocks are +// skipped, PromptTokens is the TOTAL prompt (uncached + cache read + cache +// write), the cache counters travel separately, the message id is the +// provider request id, and the openrouter-only fields stay zero. +func TestAnthropicResponseDecode(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + resp, err := Chat(context.Background(), b, "s", "u", Options{NumPredict: 5}, 5*time.Second) + if err != nil { + t.Fatalf("Chat: %v", err) + } + if resp.Message.Role != "assistant" || resp.Message.Content != "Hello" { + t.Errorf("message = %+v, want assistant/Hello (text blocks joined, thinking skipped)", resp.Message) + } + if resp.EvalCount != 7 { + t.Errorf("EvalCount = %d, want output_tokens 7", resp.EvalCount) + } + if resp.PromptTokens != 510 { + t.Errorf("PromptTokens = %d, want 10+400+100 (input_tokens is the uncached remainder only)", resp.PromptTokens) + } + if resp.CacheReadTokens != 400 || resp.CacheCreationTokens != 100 { + t.Errorf("cache counters = %d/%d, want 400/100", resp.CacheReadTokens, resp.CacheCreationTokens) + } + if resp.FinishReason != "stop" { + t.Errorf("FinishReason = %q, want end_turn mapped to stop", resp.FinishReason) + } + if resp.ProviderRequestID != "msg_01abc" { + t.Errorf("ProviderRequestID = %q, want the message id", resp.ProviderRequestID) + } + if resp.ServedModel != "" || resp.CostUSD != nil || resp.TopLogprobs != nil { + t.Errorf("openrouter-only / logprob fields must stay zero: served=%q cost=%v lp=%v", resp.ServedModel, resp.CostUSD, resp.TopLogprobs) + } + + // The cache counters reach llmlog as metadata, nothing else on the row + // changes (the model column stays the pool's pick). + entry := llmlog.Entry{Model: "pool-model"} + ApplyProviderTelemetry(&entry, resp) + if entry.Model != "pool-model" || entry.CostUSD != nil { + t.Errorf("entry model/cost disturbed: %q %v", entry.Model, entry.CostUSD) + } + if entry.Metadata["cache_read_input_tokens"] != 400 || entry.Metadata["cache_creation_input_tokens"] != 100 { + t.Errorf("metadata = %v, want both cache counters", entry.Metadata) + } + if entry.Metadata["provider_request_id"] != "msg_01abc" { + t.Errorf("metadata.provider_request_id = %v", entry.Metadata["provider_request_id"]) + } + + // A wire without cache counters leaves the metadata untouched (other + // wires' rows stay byte-identical). + plain := llmlog.Entry{} + ApplyProviderTelemetry(&plain, &ChatResponse{}) + if plain.Metadata != nil { + t.Errorf("metadata allocated without cache counters: %v", plain.Metadata) + } +} + +// TestAnthropicStopReasonMapping: the Messages API vocabulary folds into the +// one FinishReason consumers compare against (dream reads "length", the chat +// harness reads "tool_calls"); unknown values pass through raw, absent stays +// "". +func TestAnthropicStopReasonMapping(t *testing.T) { + cases := map[string]string{ + "end_turn": "stop", "stop_sequence": "stop", "pause_turn": "stop", + "max_tokens": "length", "model_context_window_exceeded": "length", + "tool_use": "tool_calls", "refusal": "content_filter", + "some_future_reason": "some_future_reason", "": "", + } + for in, want := range cases { + if got := mapAnthropicStopReason(in); got != want { + t.Errorf("map(%q) = %q, want %q", in, got, want) + } + } +} + +// TestAnthropicErrorClassification: a non-2xx answer carries its status into +// the failover classifier exactly like the other wires — 429 keeps the +// Retry-After cooldown, 529 (overloaded) fails over as a gateway fault, 401 +// is auth, 400 is request-specific, 500 stops the chain. The provider's +// error type is folded into the rendered message. +func TestAnthropicErrorClassification(t *testing.T) { + cases := []struct { + status int + errType string + hdr map[string]string + want backends.ErrClass + retry time.Duration + }{ + {429, "rate_limit_error", map[string]string{"Retry-After": "7"}, backends.ClassRateLimit, 7 * time.Second}, + {529, "overloaded_error", nil, backends.ClassGateway, 0}, + {401, "authentication_error", nil, backends.ClassAuth, 0}, + {400, "invalid_request_error", nil, backends.ClassBadRequest, 0}, + {500, "api_error", nil, backends.ClassServerFault, 0}, + } + for _, tc := range cases { + t.Run(tc.errType, func(t *testing.T) { + body := `{"type":"error","error":{"type":"` + tc.errType + `","message":"nope"}}` + rec := newAnthropicRecorder(t, tc.status, body, tc.hdr) + b := anthropicTestBackend(rec.srv.URL) + _, err := Chat(context.Background(), b, "s", "u", Options{NumPredict: 5}, 5*time.Second) + if err == nil { + t.Fatal("want an error") + } + var se *httpx.StatusError + if !errors.As(err, &se) { + t.Fatalf("error is not a *httpx.StatusError: %v", err) + } + if se.Code != tc.status || se.RetryAfter != tc.retry { + t.Errorf("StatusError = code %d retry %s, want %d/%s", se.Code, se.RetryAfter, tc.status, tc.retry) + } + if se.Body != tc.errType+": nope" { + t.Errorf("body = %q, want the provider error type folded in", se.Body) + } + if got := backends.Classify(err, backends.ProviderGeneric); got != tc.want { + t.Errorf("Classify = %v, want %v", got, tc.want) + } + }) + } +} + +// TestAnthropicSystemSplitsPromptguardRule: the per-call nonce rule the +// pipelines append to the system prompt goes into a second, unmarked block +// so the stable prefix before it can actually hit the cache (one block +// with the nonce inside would be written on every call and read never). +func TestAnthropicSystemSplitsPromptguardRule(t *testing.T) { + rule := promptguard.Rule("0123456789abcdef") + blocks := anthropicSystem("You are ctx. Answer from the sources. " + rule) + if len(blocks) != 2 { + t.Fatalf("blocks = %d, want stable prefix + rule", len(blocks)) + } + if blocks[0].Text != "You are ctx. Answer from the sources. " || blocks[0].CacheControl == nil { + t.Errorf("prefix block = %+v, want the text before the rule, cached", blocks[0]) + } + if blocks[1].Text != rule || blocks[1].CacheControl != nil { + t.Errorf("rule block = %+v, want the rule, uncached", blocks[1]) + } + + // A second call with a fresh nonce keeps the cached block byte-identical. + again := anthropicSystem("You are ctx. Answer from the sources. " + promptguard.Rule("fedcba9876543210")) + if again[0].Text != blocks[0].Text { + t.Errorf("cached prefix differs between nonces: %q vs %q", again[0].Text, blocks[0].Text) + } + + // Without a rule: one cached block, as before. A rule at position 0 + // (nothing stable before it) stays one block too. + if one := anthropicSystem("plain"); len(one) != 1 || one[0].CacheControl == nil { + t.Errorf("plain prompt = %+v", one) + } + if only := anthropicSystem(rule); len(only) != 1 { + t.Errorf("rule-only prompt = %d blocks, want 1", len(only)) + } +} + +// TestAnthropicNonStreamHasNoTopLevelCacheControl: the automatic tail +// breakpoint is a multi-turn (stream) affair; the single-turn pipelines +// keep the explicit system breakpoint only. +func TestAnthropicNonStreamHasNoTopLevelCacheControl(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + if _, err := Chat(context.Background(), anthropicTestBackend(rec.srv.URL), "s", "u", Options{NumPredict: 5}, 5*time.Second); err != nil { + t.Fatal(err) + } + _, body, _ := rec.recorded() + if _, has := body["cache_control"]; has { + t.Errorf("top-level cache_control on the non-stream path: %v", body["cache_control"]) + } +} diff --git a/go/internal/llm/chain.go b/go/internal/llm/chain.go index 5879aae7..4825440c 100644 --- a/go/internal/llm/chain.go +++ b/go/internal/llm/chain.go @@ -712,4 +712,33 @@ func ApplyProviderTelemetry(entry *llmlog.Entry, resp *ChatResponse) { } entry.Metadata["provider_request_id"] = resp.ProviderRequestID } + // Prompt-cache counters (anthropic wire): recorded only when non-zero so + // the other wires' rows stay byte-identical. prompt_tokens already holds + // the total; these make the hit rate readable per row. + applyCacheTelemetry(entry, resp.CacheReadTokens, resp.CacheCreationTokens) +} + +// applyCacheTelemetry writes the prompt-cache counters into the llmlog +// metadata. Shared by the non-streaming funnel (ApplyProviderTelemetry) and +// the web-chat stream row (chat/engine.go recordLLM). +func applyCacheTelemetry(entry *llmlog.Entry, cacheRead, cacheCreate int) { + if cacheRead == 0 && cacheCreate == 0 { + return + } + if entry.Metadata == nil { + entry.Metadata = map[string]any{} + } + if cacheRead > 0 { + entry.Metadata["cache_read_input_tokens"] = cacheRead + } + if cacheCreate > 0 { + entry.Metadata["cache_creation_input_tokens"] = cacheCreate + } +} + +// ApplyCacheTelemetry is the exported form for the stream path (chat/engine), +// which assembles its llmlog row from a StreamResult rather than a +// ChatResponse. +func ApplyCacheTelemetry(entry *llmlog.Entry, cacheRead, cacheCreate int) { + applyCacheTelemetry(entry, cacheRead, cacheCreate) } diff --git a/go/internal/llm/client.go b/go/internal/llm/client.go index e93d3f34..294f04da 100644 --- a/go/internal/llm/client.go +++ b/go/internal/llm/client.go @@ -1,7 +1,9 @@ // Package llm provides an HTTP client for LLM inference. -// Supports two wire protocols via the "protocol" parameter: +// Supports three wire protocols via the backend's Protocol field: // - "ollama": Ollama-native /api/chat (supports think, num_ctx) // - "openai": OpenAI-compatible /v1/chat/completions (works with any provider) +// - "anthropic": native Anthropic Messages API /v1/messages (anthropic.go: +// cached system prompt, output_config.effort, no sampling parameters) package llm import ( @@ -104,13 +106,21 @@ type Options struct { // provider actually said. length is the output-cap signal the dream pipeline // reads to tell a truncated answer from a malformed one. type ChatResponse struct { - Message Message - EvalCount int - PromptTokens int - FinishReason string - CostUSD *float64 - ServedModel string - ProviderRequestID string + Message Message + EvalCount int + PromptTokens int + // CacheReadTokens / CacheCreationTokens are the prompt-cache counters of + // the anthropic wire (usage.cache_read_input_tokens / + // cache_creation_input_tokens). They are ALREADY INCLUDED in + // PromptTokens — that field is the total prompt on every wire — and + // travel separately only so llmlog can record the cache hit rate + // (ApplyProviderTelemetry → metadata). Zero on every other wire. + CacheReadTokens int + CacheCreationTokens int + FinishReason string + CostUSD *float64 + ServedModel string + ProviderRequestID string // TopLogprobs are the provider's top-N alternatives for the FIRST // generated token (token text + natural-log probability), present only // when the request asked for them (Options.TopLogprobs > 0) AND the @@ -241,10 +251,14 @@ func ChatJSON(ctx context.Context, b backends.Backend, systemPrompt, userPrompt // ExtraBody are wire-relevant (OpenRouter forced zdr/deny, per-backend // attribution headers), so the loose-parameter form would silently drop them. func chatWithFormat(ctx context.Context, b backends.Backend, systemPrompt, userPrompt string, opts Options, format string, timeout time.Duration) (*ChatResponse, error) { - if b.Protocol == backends.ProtocolOpenAI { + switch b.Protocol { + case backends.ProtocolOpenAI: return chatOpenAI(ctx, b, systemPrompt, userPrompt, opts, format, timeout) + case backends.ProtocolAnthropic: + return chatAnthropic(ctx, b, systemPrompt, userPrompt, opts, format, timeout) + default: + return chatOllama(ctx, b, systemPrompt, userPrompt, opts, format, timeout) } - return chatOllama(ctx, b, systemPrompt, userPrompt, opts, format, timeout) } func chatOllama(ctx context.Context, b backends.Backend, systemPrompt, userPrompt string, opts Options, format string, timeout time.Duration) (*ChatResponse, error) { diff --git a/go/internal/llm/finishreason_wire_test.go b/go/internal/llm/finishreason_wire_test.go index 1b63ca1a..f9d11ff3 100644 --- a/go/internal/llm/finishreason_wire_test.go +++ b/go/internal/llm/finishreason_wire_test.go @@ -40,6 +40,12 @@ func TestChatDecodesFinishReason(t *testing.T) { "usage":{"completion_tokens":600,"prompt_tokens":120}}` openAINoReason = `{"choices":[{"message":{"role":"assistant","content":"hi"}}], "usage":{"completion_tokens":600,"prompt_tokens":120}}` + // The anthropic wire MAPS its vocabulary (max_tokens → length) so the + // cap consumers see it; absence still decodes to "" (anthropic.go). + anthropicWithReason = `{"id":"msg_1","content":[{"type":"text","text":"hi"}], + "stop_reason":"max_tokens","usage":{"input_tokens":120,"output_tokens":600}}` + anthropicNoReason = `{"id":"msg_1","content":[{"type":"text","text":"hi"}], + "usage":{"input_tokens":120,"output_tokens":600}}` ) tests := []struct { @@ -52,6 +58,8 @@ func TestChatDecodesFinishReason(t *testing.T) { {"ollama-absent", backends.ProtocolOllama, ollamaNoReason, ""}, {"openai-length", backends.ProtocolOpenAI, openAIWithReason, "length"}, {"openai-absent", backends.ProtocolOpenAI, openAINoReason, ""}, + {"anthropic-max_tokens", backends.ProtocolAnthropic, anthropicWithReason, "length"}, + {"anthropic-absent", backends.ProtocolAnthropic, anthropicNoReason, ""}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/go/internal/llm/stream.go b/go/internal/llm/stream.go index 190d0acd..500996fc 100644 --- a/go/internal/llm/stream.go +++ b/go/internal/llm/stream.go @@ -30,6 +30,7 @@ import ( "strings" "time" + "github.com/GottZ/ctx/internal/backends" "github.com/GottZ/ctx/internal/httpx" ) @@ -163,6 +164,15 @@ type ChatMsg struct { Content string `json:"content"` ToolCalls []ToolCall `json:"tool_calls,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"` + // ProviderBlocks are opaque content blocks the provider wants back + // unchanged ahead of the turn's text — the anthropic wire's thinking / + // redacted_thinking blocks, which the API requires on the final + // assistant turn of a tool loop. Carried IN MEMORY across the + // iterations of one turn only (the harness does not persist them; a + // later turn's history replays without them, which the API accepts for + // non-final assistant turns). json:"-": never a field of the OpenAI + // wire's message object. + ProviderBlocks []json.RawMessage `json:"-"` } // StreamEvent is handed to the ChatStream callback as the stream progresses. @@ -185,6 +195,17 @@ type StreamResult struct { PromptTokens int CompletionTokens int DraftAccept float64 // timings.draft_n_accepted/draft_n, 0 when n/a + // CacheReadTokens / CacheCreationTokens / ProviderRequestID are filled by + // the anthropic wire only (ChatResponse's twins): the cache counters are + // already inside PromptTokens and travel for llmlog metadata, the + // message id for audit. Zero on the openai wire. + CacheReadTokens int + CacheCreationTokens int + ProviderRequestID string + // ProviderBlocks are the reasoning blocks of the answer (anthropic wire), + // to be echoed back via ChatMsg.ProviderBlocks when the harness replays + // this assistant turn inside the same tool loop. nil on the openai wire. + ProviderBlocks []json.RawMessage } // StreamError is an error event delivered inside an HTTP-200 SSE stream. @@ -196,6 +217,14 @@ type StreamError struct { Message string } +// ProviderErrorType exposes the provider's error type string to the failover +// classifier (backends.Classify) without a backends→llm import: the +// anthropic wire's mid-stream error event names the same types its HTTP +// errors do (overloaded_error, rate_limit_error, …), and the classifier maps +// them onto the classes the equivalent status codes get. OpenRouter's +// numeric codes match nothing there and keep their stop-the-chain default. +func (e *StreamError) ProviderErrorType() string { return e.Code } + func (e *StreamError) Error() string { if e.Code != "" { return fmt.Sprintf("llm: stream error event: %s (code %s)", e.Message, e.Code) @@ -264,9 +293,24 @@ func buildStreamBody(model string, msgs []ChatMsg, tools []ToolDef, opts Options return json.Marshal(m) } +// ChatStreamBackend is the protocol dispatch of the streaming path: the +// anthropic wire (anthropic_stream.go) for ProtocolAnthropic, ChatStream +// (the OpenAI-compatible wire) for everything else — ollama rows included, +// which serve /v1/chat/completions too and were always streamed that way. +// The chat harness calls this; ChatStream stays exported as the openai +// implementation its tests drive directly. +func ChatStreamBackend(ctx context.Context, b backends.Backend, model string, + msgs []ChatMsg, tools []ToolDef, opts Options, timeout time.Duration, + onEvent func(StreamEvent) error) (*StreamResult, error) { + if b.Protocol == backends.ProtocolAnthropic { + return chatStreamAnthropic(ctx, b, model, msgs, tools, opts, timeout, onEvent) + } + return ChatStream(ctx, b.Host, b.APIKey, model, msgs, tools, opts, b.ExtraBody, timeout, onEvent) +} + // ChatStream sends a streaming chat request with optional tool definitions -// over the OpenAI-compatible wire (the only streaming protocol in F6) and -// assembles the response. onEvent is invoked for content fragments, tool +// over the OpenAI-compatible wire (the streaming protocol of F6; the +// anthropic wire joined via ChatStreamBackend) and assembles the response. onEvent is invoked for content fragments, tool // call starts and the finish reason; an error from onEvent aborts the // stream. Cancelling ctx aborts the in-flight generation (the HTTP request // to the backend is dropped, llama.cpp frees its slot). diff --git a/go/internal/llm/testdata/anthropic_content.sse b/go/internal/llm/testdata/anthropic_content.sse new file mode 100644 index 00000000..97900f00 --- /dev/null +++ b/go/internal/llm/testdata/anthropic_content.sse @@ -0,0 +1,36 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_01stream","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":1,"cache_read_input_tokens":300,"cache_creation_input_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"secret reasoning"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"abc"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: ping +data: {"type": "ping"} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Berlin "}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"is sunny."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":9}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/go/internal/llm/testdata/anthropic_midstream_error.sse b/go/internal/llm/testdata/anthropic_midstream_error.sse new file mode 100644 index 00000000..1a003909 --- /dev/null +++ b/go/internal/llm/testdata/anthropic_midstream_error.sse @@ -0,0 +1,12 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_01err","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":5,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"partial"}} + +event: error +data: {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}} + diff --git a/go/internal/llm/testdata/anthropic_tool_use.sse b/go/internal/llm/testdata/anthropic_tool_use.sse new file mode 100644 index 00000000..82a89322 --- /dev/null +++ b/go/internal/llm/testdata/anthropic_tool_use.sse @@ -0,0 +1,51 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_01tool","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":40,"output_tokens":2,"cache_read_input_tokens":0,"cache_creation_input_tokens":512}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sigABC"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Let me check."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01weather","name":"get_weather","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"city\": \"Ber"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"lin\", \"unit\": \"c\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":2} + +event: content_block_start +data: {"type":"content_block_start","index":3,"content_block":{"type":"tool_use","id":"toolu_02time","name":"get_time","input":{}}} + +event: content_block_stop +data: {"type":"content_block_stop","index":3} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":31}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/go/internal/llm/wire_test.go b/go/internal/llm/wire_test.go index 834d389c..488bd635 100644 --- a/go/internal/llm/wire_test.go +++ b/go/internal/llm/wire_test.go @@ -42,6 +42,10 @@ func newWireRecorder(t *testing.T, content string) *wireRecorder { fmt.Fprintf(w, `{"choices":[{"message":{"role":"assistant","content":%q}}],"usage":{"completion_tokens":1,"prompt_tokens":1}}`, content) return } + if r.URL.Path == "/v1/messages" { + fmt.Fprintf(w, `{"id":"msg_1","content":[{"type":"text","text":%q}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`, content) + return + } fmt.Fprintf(w, `{"message":{"role":"assistant","content":%q},"eval_count":1,"prompt_eval_count":1}`, content) })) t.Cleanup(rec.srv.Close) @@ -70,6 +74,8 @@ var wireCases = []struct { }{ {"ollama", backends.ProtocolOllama, "/api/chat", `"options"`}, {"openai", backends.ProtocolOpenAI, "/v1/chat/completions", `"max_tokens"`}, + // anthropic: output_config is a Messages-API-only field. + {"anthropic", backends.ProtocolAnthropic, "/v1/messages", `"output_config"`}, } func assertWire(t *testing.T, rec *wireRecorder, wantPath, bodyMark string) { @@ -167,6 +173,9 @@ func TestChatJSONWirePath(t *testing.T) { jsonMarks := map[backends.Protocol]string{ backends.ProtocolOllama: `"format":"json"`, backends.ProtocolOpenAI: `"response_format"`, + // No JSON-mode marker on the anthropic wire (documented no-op — + // output_config.format needs a schema); the path is the assertion. + backends.ProtocolAnthropic: `"max_tokens"`, } for _, tc := range wireCases { t.Run(tc.name, func(t *testing.T) { diff --git a/go/internal/schemacontract/manifest.json b/go/internal/schemacontract/manifest.json index c055bea1..c92d0413 100644 --- a/go/internal/schemacontract/manifest.json +++ b/go/internal/schemacontract/manifest.json @@ -1,8 +1,8 @@ { "manifest_version": 1, "generated_against": { - "migration_max": 154, - "migration_count": 42, + "migration_max": 155, + "migration_count": 43, "pg_major": 18, "image": "pgvector-timescaledb:pg18" }, diff --git a/go/internal/store/embed_failures.go b/go/internal/store/embed_failures.go index 4d54d09f..2c111372 100644 --- a/go/internal/store/embed_failures.go +++ b/go/internal/store/embed_failures.go @@ -44,6 +44,17 @@ const ( // memo-free "stays unembedded, retried every cycle" semantics for this // case — a pool outage there must not permanently park blocks). EmbedFailureSensitivityIneligible EmbedFailureClass = "sensitivity_ineligible" + // EmbedFailureCallerTimeout marks a block whose embed did not fit the + // query path's sync budget (embed_backfill.sync_budget) or died with + // the request context — the backend is not known to be down, it was + // slow relative to THIS caller's deadline (the tracking issue, force-push.me/damienmoon/ctx/issues/1: a + // 10k-token block on a CPU backend). The memo therefore says nothing + // about the backend and must not park the block on the wire curve: the + // query path (EmbedFailureQueryPathExcludedPredicate) never picks a + // caller_timeout block again, the background arm picks it at once + // (next_attempt_at = now()). A later wire failure in the background + // arm reclassifies it as wire with the ordinary backoff. + EmbedFailureCallerTimeout EmbedFailureClass = "caller_timeout" ) // maxLastErrorLen mirrors migration 113's documented last_error contract: @@ -153,6 +164,27 @@ func RecordEmbedFailure(ctx context.Context, q pgxdb.Execer, blockID string, cla return nil } + if class == EmbedFailureCallerTimeout { + // Eligible for the background arm at once: the backoff curve is + // for a backend that failed, this backend only took longer than + // the caller could wait. Pfad A's own exclusion of this class is + // in the predicate, not in the timestamp. + _, err := q.Exec(ctx, + `INSERT INTO context_embed_failures (block_id, migration_id, attempts, last_error, last_class, next_attempt_at) + VALUES ($1, NULL, 1, $2, $3, now()) + ON CONFLICT (block_id) WHERE migration_id IS NULL + DO UPDATE SET attempts = context_embed_failures.attempts + 1, + last_error = EXCLUDED.last_error, + last_class = EXCLUDED.last_class, + next_attempt_at = now()`, + blockID, normalizedErr, string(class), + ) + if err != nil { + return fmt.Errorf("store: record embed failure (caller_timeout): %w", err) + } + return nil + } + baseSecs := backoffBase.Seconds() capSecs := backoffCap.Seconds() _, err := q.Exec(ctx, @@ -265,3 +297,21 @@ const EmbedFailureExcludedPredicate = ` AND f.migration_id IS NULL AND f.next_attempt_at > now() )` + +// EmbedFailureQueryPathExcludedPredicate is Pfad A's (query.go +// backfillPending) variant of EmbedFailureExcludedPredicate: the same +// backoff exclusion PLUS every block memoized as caller_timeout, for good. +// A block that once overran the sync budget is the background arm's job +// from then on; re-picking it inline would burn the budget of every later +// query for a block the arm is about to embed anyway (and, while the arm +// holds its FOR UPDATE row, embed it a second time in parallel). The +// background arm keeps the plain predicate, so for it a caller_timeout memo +// (next_attempt_at = now()) is immediately eligible. Same positional +// contract as its sibling: bare context_blocks as the outer FROM. +const EmbedFailureQueryPathExcludedPredicate = ` + AND NOT EXISTS ( + SELECT 1 FROM context_embed_failures f + WHERE f.block_id = context_blocks.id + AND f.migration_id IS NULL + AND (f.next_attempt_at > now() OR f.last_class = 'caller_timeout') + )` diff --git a/go/migrations/155_backend_protocol_anthropic.sql b/go/migrations/155_backend_protocol_anthropic.sql new file mode 100644 index 00000000..8318b23e --- /dev/null +++ b/go/migrations/155_backend_protocol_anthropic.sql @@ -0,0 +1,41 @@ +-- ============================================================================= +-- 155_backend_protocol_anthropic.sql — 'anthropic' joins the backend wire +-- protocols +-- Part of ctx by GottZ (https://github.com/GottZ/ctx) +-- ============================================================================= +-- The baseline (113, context_backends) pins the protocol column to the three +-- wires the daemon spoke when the pool was designed: openai, ollama, rerank. +-- The native Anthropic Messages API wire (go/internal/llm/anthropic.go) is a +-- fourth; without this migration the CHECK rejects the row the handler's +-- validation already accepted, and the create fails one layer too late with +-- a bare constraint error. +-- +-- The constraint was declared inline and unnamed in the baseline, so its name +-- is whatever the server assigned (context_backends_protocol_check on every +-- PostgreSQL this project supports). The DO block resolves it from the catalog +-- by table + definition instead of trusting the derived name, so a database +-- whose constraint was ever renamed by hand migrates the same way. +-- +-- Nothing else changes: the column keeps its DEFAULT 'openai', the schema +-- manifest records column types only (no CHECK clauses), and existing rows are +-- untouched. Rolling back the daemon to a build without the anthropic wire +-- leaves such a row in the table; it would then dispatch onto the Ollama path +-- (pre-155 fallthrough) — delete or disable anthropic rows before downgrading. + +DO $$ +DECLARE + con_name text; +BEGIN + SELECT conname INTO con_name + FROM pg_constraint + WHERE conrelid = 'context_backends'::regclass + AND contype = 'c' + AND pg_get_constraintdef(oid) LIKE '%protocol%'; + IF con_name IS NOT NULL THEN + EXECUTE format('ALTER TABLE context_backends DROP CONSTRAINT %I', con_name); + END IF; + ALTER TABLE context_backends + ADD CONSTRAINT context_backends_protocol_check + CHECK (protocol IN ('openai','ollama','rerank','anthropic')); +END +$$; diff --git a/go/web/src/lib/backends.ts b/go/web/src/lib/backends.ts index 1bf2957a..f41a2081 100644 --- a/go/web/src/lib/backends.ts +++ b/go/web/src/lib/backends.ts @@ -64,7 +64,7 @@ export function isTrustElevation(prev: string, next: string): boolean { // Wire enums (verified against backends/validate.go, NOT the design doc which // undercounted locality). locality is also auto-derived from base_url on create // and cross-checked against the host, so '' (auto) is the sane default. -export const PROTOCOLS = ['openai', 'ollama', 'rerank'] as const +export const PROTOCOLS = ['openai', 'ollama', 'anthropic', 'rerank'] as const export const PROVIDER_CLASSES = ['generic', 'llamacpp', 'openrouter'] as const export const LOCALITIES = ['local', 'lan', 'external'] as const diff --git a/go/web/src/lib/settings.test.ts b/go/web/src/lib/settings.test.ts index 17c8651c..e308ec1b 100644 --- a/go/web/src/lib/settings.test.ts +++ b/go/web/src/lib/settings.test.ts @@ -41,7 +41,7 @@ describe('widgetFor', () => { describe('selectOptions', () => { it('mirrors the Go enums', () => { - expect(selectOptions('protocol')).toEqual(['ollama', 'openai']) + expect(selectOptions('protocol')).toEqual(['ollama', 'openai', 'anthropic']) expect(selectOptions('think')).toEqual(['true', 'false']) expect(selectOptions('string')).toEqual([]) }) diff --git a/go/web/src/lib/settings.ts b/go/web/src/lib/settings.ts index b437515f..df332666 100644 --- a/go/web/src/lib/settings.ts +++ b/go/web/src/lib/settings.ts @@ -43,7 +43,7 @@ export function widgetFor(type: string): Widget { export function selectOptions(type: string): string[] { switch (type) { case 'protocol': - return ['ollama', 'openai'] + return ['ollama', 'openai', 'anthropic'] case 'think': return ['true', 'false'] case 'sensitivity': diff --git a/go/web/src/routes/settings/backends/ModelMapEditor.svelte b/go/web/src/routes/settings/backends/ModelMapEditor.svelte index 55868d8e..294d6c58 100644 --- a/go/web/src/routes/settings/backends/ModelMapEditor.svelte +++ b/go/web/src/routes/settings/backends/ModelMapEditor.svelte @@ -134,7 +134,7 @@ {#if invalid[i]} invalid JSON — last valid state is kept {:else} - JSON object, sent verbatim as model_map params (empty = none). Unknown keys reach the wire on OpenAI-protocol backends. + JSON object, sent verbatim as model_map params (empty = none). Unknown keys reach the wire on OpenAI- and Anthropic-protocol backends (on Anthropic, effort becomes output_config.effort). {/if}