From 8e11ca79d40f83fb5f4c94d145a78927dd65530e Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Fri, 9 Oct 2026 03:51:08 +0200 Subject: [PATCH 01/16] fix(backfill): dedicated embed-backfill arm and a time budget for the query path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps showed up on a CPU embed backend (Qwen3-Embedding-8B, the documented ~23 tok/s) with a ~10k-token block (issue damienmoon/ctx#1): 1. backfillOneEmbedding was called only at the top of runDreamLoop, so with dream.enabled=false (the default) or Dream switched off at runtime nothing embedded pending blocks in the background. runEmbedBackfillLoop is now its own goroutine, started regardless of Dream: a successful pick loops at once, an empty pick waits embed_backfill.interval (default 15 s), 0 = arm off, polled for a hot re-enable. The dream loop keeps its inline call; the tx-wrapped FOR UPDATE SKIP LOCKED pick makes two pickers safe (Welle-49). 2. backfillPending had a count cap (sync_cap) but no time budget: one slow block consumed the caller's whole deadline (120 s on the CLI client), the question's own embed was never admitted and the query answered 500, and the failure was memoized as class wire although the backend was only slow for this caller. embed_backfill.sync_budget (seconds, default 30, 0 = unbounded) bounds the whole pre-search loop per request; each wire call runs under what is left. A block that overruns it, or dies with the request context, is memoized with the new class caller_timeout: next_attempt_at = now(), the query path never picks it again (EmbedFailureQueryPathExcludedPredicate), the background arm picks it at once, and a later wire failure there reclassifies it as wire with the ordinary backoff. Also: the "german detected" log line is now "non-english heuristic matched" (the heuristic fires on Swedish, Danish and Finnish too; the translation was right, the label was not). Tests (Rot/Grün through the production functions): - handler: TestBackfillPending_SyncBudget_Integration — SyncBudget=0 holds the request for the backend's full delay (red); SyncBudget=300ms returns inside the budget with the parent context intact, the backend sees the cancel, the memo is caller_timeout and due now, a second backfillPending makes no wire call, the background predicate still sees the block; a request cancel mid-embed classifies caller_timeout, not wire. - events: TestEmbedBackfillArm_RunsWithoutDream_Integration — the arm embeds a pending block with DreamEnabled=false, picks a caller_timeout block at once, and Interval=0 turns it off. - compose and description gates: TestComposeDeclaresEveryRegistryKey and TestKeyDescriptionsComplete pass with the two new keys. Run on 2026-10-09 in golang:1.27-alpine with CTX_TEST_PG_IMAGE=localhost/ ctx-db:38fdb74f: go build ./..., go vet -tags=integration, go test -short for config/store/handler/events (one pre-existing failure, TestRebuildViaWorker_ TimeoutKillsHangingChild, fails identically on the unpatched checkout in this container), and the backfill integration suites of handler, events and store all green. Co-Authored-By: Claude Fable 5.1 --- docker-compose.yml | 4 +- docs/operations.md | 2 + go/internal/config/config.go | 23 ++ go/internal/config/descriptions.go | 2 + .../events/backfill_arm_integration_test.go | 124 ++++++++++ go/internal/events/scheduler.go | 80 ++++++ .../backfill_syncbudget_integration_test.go | 230 ++++++++++++++++++ go/internal/handler/query.go | 54 +++- go/internal/store/embed_failures.go | 50 ++++ 9 files changed, 564 insertions(+), 5 deletions(-) create mode 100644 go/internal/events/backfill_arm_integration_test.go create mode 100644 go/internal/handler/backfill_syncbudget_integration_test.go diff --git a/docker-compose.yml b/docker-compose.yml index e0ab9aa7..6fd69c8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -362,8 +362,10 @@ services: # ── status.* (1) ── CTX_STATUS_CHANNEL_PROBE_INTERVAL: ${CTX_STATUS_CHANNEL_PROBE_INTERVAL:-} - # ── embed_backfill.* (4) ── + # ── embed_backfill.* (6) ── CTX_EMBED_BACKFILL_SYNC_CAP: ${CTX_EMBED_BACKFILL_SYNC_CAP:-} + CTX_EMBED_BACKFILL_SYNC_BUDGET: ${CTX_EMBED_BACKFILL_SYNC_BUDGET:-} + CTX_EMBED_BACKFILL_INTERVAL: ${CTX_EMBED_BACKFILL_INTERVAL:-} CTX_EMBED_BACKFILL_MAX_TOKENS: ${CTX_EMBED_BACKFILL_MAX_TOKENS:-} CTX_EMBED_BACKFILL_BACKOFF_BASE: ${CTX_EMBED_BACKFILL_BACKOFF_BASE:-} CTX_EMBED_BACKFILL_BACKOFF_CAP: ${CTX_EMBED_BACKFILL_BACKOFF_CAP:-} diff --git a/docs/operations.md b/docs/operations.md index 487c31bc..b57b1a7b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -477,6 +477,8 @@ The `llama-embed` sidecar (`docker-compose.override.yml`, Qwen3-Embedding-8B, `- **Backfill failure memo (migration 113, Achse 04 W04-2).** Both backfill paths — the scheduler arm (`backfillOneEmbedding`) and the query-path pre-search loop (`backfillPending`) — used to retry a NULL-embedding block **indefinitely** if it exceeded the embed window: no truncate, no skip, nothing written to the DB on failure, so the next pick landed on the exact same block (observed 2026-07-10: a 13k-char block at 4704 tokens against a 4096 window looped every ~20 s and blocked ~265 pending blocks behind it). This is closed structurally, not operationally: every failed embed attempt now upserts a row into `context_embed_failures` (`block_id`, `attempts`, `last_error` — normalized, ≤500 chars, never raw wire/embed-text content —, `last_class`, `next_attempt_at`), and both pick queries exclude a block whose `next_attempt_at` is still in the future. A non-oversize failure (backend down, transient wire error) gets an exponential backoff (`embed_backfill.backoff_base` × 2^(attempts-1), capped at `embed_backfill.backoff_cap`; defaults 1min/24h; the base must be > 0 — V21 rejects 0, which would mean "retry immediately" and a tight loop against a failing backend, issue #38) — the next PEEK reaches the next-oldest block instead of re-picking the same one. An oversize block — classified either by a conservative pre-wire token estimate (`len(text)/4` against `embed_backfill.max_tokens`, default 24000, chosen to sit under the live kv-unified pool's 32k minus interactive headroom — see the tuning section above) or by an HTTP 400 whose body names `exceed_context_size` — gets `next_attempt_at = 'infinity'` instead: permanently parked, never a blind 24h-in-slow-motion retry, but never silently dropped either (the memo row stays visible in `context_embed_failures`). Known limitation: `ClearEmbedding` does NOT currently clear the block's failure memo, so a content edit that shrinks a previously oversize-parked block below the token threshold does **not** reactivate it on its own — the `infinity` exclusion persists until the memo row is cleaned up (a janitor for this is not yet built; see the Achse-04 backlog). The query-path loop additionally carries a synchronous cap (`embed_backfill.sync_cap`, default 4): before this wave a large rest-transient (e.g. after a bulk ingest that outran the scheduler arm) made the *first* interactive query pay the entire nachzug inline; now it embeds at most `sync_cap` blocks per request and leaves the rest to the scheduler arm. Keeping blocks compact (~1–1.5k chars per the store guidance) or sizing the embed window above your largest block is still the actual fix for an oversize block's content — the memo mechanism only stops it from starving everything behind it. +**The dedicated backfill arm and the query-path time budget (issue damienmoon/ctx#1).** Two gaps in the above showed up on a CPU embed backend (Qwen3-Embedding-8B at the documented ~23 tok/s) with a ~10k-token block. First, the scheduler arm ran only at the top of the dream loop, so with `dream.enabled=false` — the default — nothing backfilled pending embeddings in the background at all; the query path was the only path. `runEmbedBackfillLoop` is now its own goroutine, started regardless of Dream: a successful pick loops at once, an empty pick waits `embed_backfill.interval` (default 15 s), `0` turns the arm off (polled every 15 s for a hot re-enable). The dream loop keeps its inline call; the tx-wrapped `FOR UPDATE SKIP LOCKED` pick makes two pickers safe. Second, `sync_cap` bounds how *many* blocks a query embeds inline, not how *long*: one block that needs 7 minutes consumed the caller's whole 120 s deadline, the question's own embed was never admitted, and the query answered 500 instead of degrading. `embed_backfill.sync_budget` (seconds, default 30, `0` = unbounded) now bounds the whole pre-search loop per request; each wire call runs under what is left of it. A block that overruns the budget — or dies with the request context — is memoized with the class `caller_timeout`: the query path never picks it again (`EmbedFailureQueryPathExcludedPredicate`), the background arm picks it at once (`next_attempt_at = now()`), and a later wire failure there reclassifies it as `wire` with the ordinary backoff. The block stays FTS-visible meanwhile. Previously such a failure was memoized as `wire` and re-burned the deadline of every query that landed after the backoff lapsed. + **Backfill skips retrieval-excluded types (2026-07-25).** Both backfill paths now also skip a block whose *type* carries retrieval policy `excluded` in the block-type registry (shipped builtins: `checkpoint`, `system-meta`). Such a vector has no consumer at all — every ranked path filters against the registry's visible-type allowlist, so the embedding was written once and then read by nobody. The trigger was a live incident on 2026-07-25 with two independent arms: 33 Hermes "Compaction source" parts (`type=checkpoint`, ~36.5 kB each) saturated the CPU embed backend for hours, and the digest rebuild rewrites `topic-map-private` (`type=system-meta`, 73.7 kB ≈ ~32k real tokens) on **every** boot, which clears its embedding and bought ~60 min of CPU embed for a dead vector — recurring, forever. The filter is policy-driven, not a hardcoded type list (same mechanic as `test.sh` T19), and deliberately conservative in the fail-safe direction: only `_global`-`excluded` skips; `damped` and `aggregate-to-parent` stay embeddable, a block whose type has no registry row at all stays embeddable, and a tenant scope that overrides an excluded type to a visible policy re-opens that type for the backfill (the pick has no reader identity, so the reader-aware overlay rule is mirrored as the union over all possible readers). Two operational consequences: blocks of an excluded type keep `embedding IS NULL` forever — that is the intended end state, not a stuck queue — and the `/status` `embed_backlog` metric still counts them, so a corpus with such blocks shows a permanent backlog floor (live at the time of writing: 24 `checkpoint` + 1 `system-meta`). Existing `infinity` failure memos for such blocks become moot: the predicate no longer even sees the rows. Flipping a type's policy from `excluded` to a visible one makes its blocks embeddable again on the next cycle without any manual step. **Re-embed migration worker (Achse 04 W04-4).** With an active migration in `running` or `verifying`, a dedicated scheduler arm (`migrateOneEmbedding`, BACKGROUND class, defers to interactive demand) walks all blocks that carry an old-space vector (`embedding IS NOT NULL AND embedding_next IS NULL AND NOT is_archived`) oldest-first from a persistent cursor, embeds them via the model-map key `embed_next` on the regular embed chain, and writes `embedding_next`/`embed_model_next` — the serving pair stays byte-identical until the cutover. Operator knobs: `embed_migration.batch_per_cycle` (default 8, `<=0` disables the arm), `embed_migration.max_tokens` (default 24000, pre-wire oversize gate; `0` disables the estimate — the wire classification of HTTP 400 `exceed_context_size` stays), `embed_migration.backoff_base`/`backoff_cap` (defaults 1min/24h; the base must be > 0 — V21, same tight-loop reasoning as the backfill sibling) — all hot, global-only. Failure memos are migration-scoped rows in `context_embed_failures` (independent from the regular backfill's memos): transient wire errors back off exponentially, `oversize` and `sensitivity_ineligible` (no backend trusted for the block's sensitivity) park at `infinity` — visible in the memo table, surfaced in status/state.sh with W04-7. **If a migration sits in `paused` with `last_error = "model guard: resolved , expected "`**: the target backend's model_map lost (or never had) the `embed_next` key resolving to the migration's `to_model` — fix the backend row, then resume; the guard re-checks every cycle and exists because a missing key silently falls back to the OLD model (relabeling old vectors as migrated). The arm maintains a runtime partial index `idx_embedding_next_pending` (created `CONCURRENTLY` when a migration is active, INVALID leftovers from a crashed build are dropped and rebuilt automatically, dropped again once the migration is terminal) — seeing it in the catalog is normal only while a migration exists. Measured backend behavior (2026-07-24, CPU llama.cpp, kv-unified): a parallel embed that overflows the shared pool WAITS (no error path), and even below the pool limit a small interactive embed queues behind a large prefill — at ~20 tok/s CPU prefill a 24k-token block holds a slot for ~19 min, which is what `max_tokens` actually bounds. diff --git a/go/internal/config/config.go b/go/internal/config/config.go index 55687c04..71cf7a69 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -1592,6 +1592,29 @@ type EmbedBackfillConfig struct { // unit test that never sets this field disables the cap rather than // silently blocking every backfillPending call. SyncCap int `key:"embed_backfill.sync_cap" env:"CTX_EMBED_BACKFILL_SYNC_CAP" default:"4" mut:"hot" tenancy:"global-only"` + // SyncBudget bounds Pfad A in SECONDS, next to SyncCap's count: the + // whole pre-search loop of one request runs inside this wall-clock + // budget, and each block's embed call runs under what is left of it. + // SyncCap alone could not stop one slow block from eating the caller's + // entire deadline (issue damienmoon/ctx#1: a ~10k-token block against a + // CPU embed backend at ~23 tok/s needs ~7 min, the CLI client's deadline + // is 120 s — the question's own embed was never admitted and the query + // answered 500 instead of degrading). A block that overruns the budget + // is memoized as caller_timeout (store.EmbedFailureCallerTimeout) and + // left to the background arm; the question embed keeps the rest of the + // deadline. Default 30 s (a quarter of the CLI deadline). 0 = unbounded, + // the pre-budget behavior (explicit opt-out, SyncCap convention). + SyncBudget time.Duration `key:"embed_backfill.sync_budget" env:"CTX_EMBED_BACKFILL_SYNC_BUDGET" default:"30" mut:"hot" tenancy:"global-only"` + // Interval is the idle wait in seconds of the dedicated background + // backfill arm (runEmbedBackfillLoop) between two empty picks. Until + // this key the only background backfill was the call at the top of the + // dream loop, so with dream.enabled=false (the default) NOTHING embedded + // pending blocks in the background and Pfad A was the only path (issue + // damienmoon/ctx#1). The arm runs regardless of Dream; a successful + // pick loops at once, an empty pick waits this long. 0 = arm off + // (polled every 15 s for a hot re-enable, contract.recheck_interval + // convention). + Interval time.Duration `key:"embed_backfill.interval" env:"CTX_EMBED_BACKFILL_INTERVAL" default:"15" mut:"hot" tenancy:"global-only"` // MaxTokens is the pre-wire Oversize-Gate estimate threshold // (design/04 §4.4): len(embedText)/4 above this skips the block WITHOUT // a wire call (last_class='oversize', next_attempt_at='infinity' — diff --git a/go/internal/config/descriptions.go b/go/internal/config/descriptions.go index 3b653642..95ef754a 100644 --- a/go/internal/config/descriptions.go +++ b/go/internal/config/descriptions.go @@ -175,6 +175,8 @@ var keyDescriptions = map[string]string{ //nolint:gosec // G101-Fehlalarm: Schl "contract.recheck_interval": "Seconds between periodic schema-drift re-checks; 0 runs no checks while the ticker keeps polling so a later hot enable takes effect", "status.channel_probe_interval": "Seconds between latency probes of the four retrieval channels (semantic, fts_de, fts_en, trigram); 0 (default) disables the probe", "embed_backfill.sync_cap": "Maximum pending blocks a query embeds inline before its search runs; 0 = uncapped, the cap bounds first-query latency after a backlog", + "embed_backfill.sync_budget": "Seconds a query may spend embedding pending blocks before its own search; a block that overruns it is memoized as caller_timeout and left to the background arm, 0 = unbounded", + "embed_backfill.interval": "Seconds the background embed-backfill arm waits between two empty picks; it runs regardless of Dream, 0 = arm off (polled for a hot re-enable)", "embed_backfill.max_tokens": "Estimated-token threshold (length/4) above which a block is permanently parked as oversize without a wire call; 0 disables the pre-check", "embed_backfill.backoff_base": "Base delay in seconds of the exponential retry curve (base * 2^(attempts-1)) for transient embed failures. Must be > 0 (V21) — 0 would mean retry immediately, a tight loop against a failing embed backend", "embed_backfill.backoff_cap": "Upper bound in seconds on the embed-failure retry delay; a persistently failing block retries at most this often", diff --git a/go/internal/events/backfill_arm_integration_test.go b/go/internal/events/backfill_arm_integration_test.go new file mode 100644 index 00000000..9c223a12 --- /dev/null +++ b/go/internal/events/backfill_arm_integration_test.go @@ -0,0 +1,124 @@ +//go:build integration + +// Rot-Gate for issue damienmoon/ctx#1, scheduler half: the embed-backfill +// arm (backfillOneEmbedding) was called ONLY at the top of runDreamLoop, so +// with dream.enabled=false — the registry default — no goroutine ever +// embedded a pending block in the background. Live: a block stayed pending +// for two hours while the scheduler logged guard and digest ticks only; the +// query path was the sole embedder, inside its caller's deadline. +// +// Rot is the premise, stated against the production wiring: a scheduler +// with DreamEnabled=false starts no dream worker, and nothing else in Run +// called backfillOneEmbedding before this wave. Grün is the dedicated arm +// (runEmbedBackfillLoop): started regardless of Dream, it embeds the pending +// block within a few intervals, and a caller_timeout memo (the query path's +// hand-off) does not hold it back. +// +// Run: `go test -tags=integration ./internal/events/ -run TestEmbedBackfillArm -count=1 -v`. +package events + +import ( + "context" + "testing" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/config" + "github.com/GottZ/ctx/internal/dispatch" + "github.com/GottZ/ctx/internal/store" + "github.com/GottZ/ctx/internal/testdb" +) + +func TestEmbedBackfillArm_RunsWithoutDream_Integration(t *testing.T) { + if testing.Short() { + t.Skip("integration test") + } + pool := testdb.SetupTestDB(t) + ctx := context.Background() + + srv := headOfLineEmbedServer(t) + bpool := backends.NewPool(nil, nil) + bpool.SeedSnapshotForTest([]backends.Backend{embedPoolRow("embed-a", srv.URL, 100)}) + d := dispatch.New(nil, dispatch.DefaultSettings()) // empty policy: pass-through + t.Cleanup(d.Close) + + cfg := &config.Config{EmbedBackfill: config.EmbedBackfillConfig{ + Interval: 100 * time.Millisecond, + MaxTokens: 0, + BackoffBase: time.Hour, + BackoffCap: 24 * time.Hour, + }} + // DreamEnabled=false: the RED premise — before this wave, Run started no + // goroutine that could reach backfillOneEmbedding in this configuration. + s := NewScheduler(pool, config.NewStore(cfg), bpool, StartupConfig{DreamEnabled: false}) + s.SetDispatcher(d) + + waitEmbedded := func(t *testing.T, within time.Duration) bool { + t.Helper() + deadline := time.Now().Add(within) + for time.Now().Before(deadline) { + if pendingCount(t, pool) == 0 { + return true + } + time.Sleep(25 * time.Millisecond) + } + return false + } + + t.Run("arm_embeds_a_pending_block_with_dream_off", func(t *testing.T) { + seedPendingBlock(t, pool, "arm-pending-normal", time.Hour) + defer clearBlocks(t, pool) + + armCtx, stop := context.WithCancel(ctx) + defer stop() + go s.runEmbedBackfillLoop(armCtx) + + if !waitEmbedded(t, 5*time.Second) { + t.Fatalf("block still pending after 5 s: the dedicated arm did not embed it (pending=%d)", pendingCount(t, pool)) + } + }) + + t.Run("arm_picks_a_caller_timeout_block_at_once", func(t *testing.T) { + seedPendingBlock(t, pool, "arm-handed-off", time.Hour) + defer clearBlocks(t, pool) + + // The query path's hand-off: a caller_timeout memo, due now. + var blockID string + if err := pool.QueryRow(ctx, `SELECT id FROM context_blocks WHERE title = 'arm-handed-off'`).Scan(&blockID); err != nil { + t.Fatalf("read block id: %v", err) + } + if err := store.RecordEmbedFailure(ctx, pool, blockID, store.EmbedFailureCallerTimeout, + store.NormalizeEmbedError(store.EmbedFailureCallerTimeout, "context deadline exceeded"), + cfg.EmbedBackfill.BackoffBase, cfg.EmbedBackfill.BackoffCap); err != nil { + t.Fatalf("record caller_timeout memo: %v", err) + } + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM context_embed_failures WHERE block_id = $1`, blockID) }) + + armCtx, stop := context.WithCancel(ctx) + defer stop() + go s.runEmbedBackfillLoop(armCtx) + + if !waitEmbedded(t, 5*time.Second) { + t.Fatalf("caller_timeout block still pending after 5 s: the arm must pick it at once, not after a backoff") + } + }) + + t.Run("interval_zero_turns_the_arm_off", func(t *testing.T) { + seedPendingBlock(t, pool, "arm-off-pending", time.Hour) + defer clearBlocks(t, pool) + + off := *cfg + off.EmbedBackfill.Interval = 0 + sOff := NewScheduler(pool, config.NewStore(&off), bpool, StartupConfig{DreamEnabled: false}) + sOff.SetDispatcher(d) + + armCtx, stop := context.WithCancel(ctx) + defer stop() + go sOff.runEmbedBackfillLoop(armCtx) + + time.Sleep(600 * time.Millisecond) + if got := pendingCount(t, pool); got != 1 { + t.Fatalf("pending = %d, want 1 (Interval=0 is the explicit off switch)", got) + } + }) +} diff --git a/go/internal/events/scheduler.go b/go/internal/events/scheduler.go index ee3e2686..8edbfc78 100644 --- a/go/internal/events/scheduler.go +++ b/go/internal/events/scheduler.go @@ -782,6 +782,14 @@ func (s *Scheduler) Run(ctx context.Context) { embedMigrateTicker := time.NewTicker(embedMigrateInterval) defer embedMigrateTicker.Stop() + // The dedicated embed-backfill arm (issue damienmoon/ctx#1): pending + // embeddings used to be backfilled only at the top of the dream loop, so + // with dream.enabled=false (the default) or Dream switched off at runtime + // nothing embedded them in the background. The arm runs regardless of + // Dream; the dream loop keeps its inline call, the tx-wrapped + // FOR UPDATE SKIP LOCKED pick makes two pickers safe (Welle-49). + go s.runEmbedBackfillLoop(ctx) + // Dream runs in its own goroutine(s) as continuous loop(s). // DreamParallelism workers all share the same DB; PickBlock's FOR UPDATE // SKIP LOCKED ensures distinct blocks per worker. Backfill stays single- @@ -2043,6 +2051,78 @@ func (t *txTail) done() { } } +// backfillArmPoll is how often the dedicated embed-backfill arm re-reads its +// config while embed_backfill.interval is 0 (arm off), so a hot re-enable +// takes effect without a restart (contract.recheck_interval convention). +const backfillArmPoll = 15 * time.Second + +// runEmbedBackfillLoop is the dedicated background arm for pending +// embeddings (issue damienmoon/ctx#1). It is the same work as the call at the +// top of runDreamLoop — backfillOneEmbedding under the iterated tenant's +// snapshot and router — on its own cadence: a successful pick loops at once, +// an empty pick waits embed_backfill.interval, an error waits 10 s. Under +// !enforcing() it yields to interactive demand exactly like the dream loop +// (P-Fallback); under Enforcing the wire call waits at its target in its +// background lease. Interval <= 0 turns the arm off; it then polls the +// config every backfillArmPoll so a hot enable needs no restart. +func (s *Scheduler) runEmbedBackfillLoop(ctx context.Context) { + slog.Info("scheduler: embed backfill arm started", "poll", backfillArmPoll) + var tenantCursor uint64 + wait := func(d time.Duration) bool { + select { + case <-ctx.Done(): + return false + case <-time.After(d): + return true + } + } + for ctx.Err() == nil { + tenants := s.backgroundTenantsFn(ctx) + if len(tenants) == 0 { + tenants = []backgroundTenant{{scope: store.GlobalScope}} + } + bt := tenants[tenantCursor%uint64(len(tenants))] + tenantCursor++ + cfg := s.cfg.SnapshotForTenant(ctx, bt.scope) + + interval := cfg.EmbedBackfill.Interval + if interval <= 0 { + if !wait(backfillArmPoll) { + return + } + continue + } + for !s.enforcing() && s.interactiveDemand() > 0 { + if !wait(dreamYieldWait) { + return + } + } + + backfilled, err := s.backfillArmOnce(ctx, cfg, bt.scope) + switch { + case err != nil: + slog.Error("scheduler: embed backfill arm error", "error", err) + if !wait(10 * time.Second) { + return + } + case backfilled: + continue // Loop at once: more may be pending. + default: + if !wait(interval) { + return + } + } + } +} + +// backfillArmOnce is one pick of the dedicated arm with its own panic guard, +// so a panic in one cycle is logged and the arm goes on (the dream loop's +// runDreamCycle has the same shape). +func (s *Scheduler) backfillArmOnce(ctx context.Context, cfg *config.Config, tenant string) (bool, error) { + defer guardPanic("embed backfill arm") + return s.backfillOneEmbedding(ctx, s.newRouter(cfg, tenant), cfg) +} + // backfillOneEmbedding finds one block with missing embedding, generates it, and stores it. // Returns true if a block was backfilled, false if none needed. // The embed chains over the pool (G28): role dream-embed when configured, diff --git a/go/internal/handler/backfill_syncbudget_integration_test.go b/go/internal/handler/backfill_syncbudget_integration_test.go new file mode 100644 index 00000000..077115ce --- /dev/null +++ b/go/internal/handler/backfill_syncbudget_integration_test.go @@ -0,0 +1,230 @@ +//go:build integration + +// Rot-Gate for issue damienmoon/ctx#1: the query-path pre-search backfill +// (backfillPending) had a COUNT cap (embed_backfill.sync_cap) but no TIME +// budget. One block whose embed could not finish inside the caller's +// deadline consumed the entire deadline (live: a ~10k-token block against a +// CPU embed backend at ~23 tok/s needs ~7 min, the CLI client's deadline is +// 120 s), the question's own embed was never admitted ("embedcache: +// admission: context canceled") and the query answered 500. The failure was +// memoized as class `wire` — backend-down semantics for a backend that was +// healthy, just slow for this caller — so every query that landed after the +// backoff lapsed repeated the burn. +// +// Rot and Grün run through the SAME production function. SyncBudget=0 is BY +// CONSTRUCTION the historical shape (no deadline on the wire call, no +// budget check at the loop head); SyncBudget>0 is the fix. +// +// SyncBudget=0 (Ist): the slow block holds the loop for its full embed time. +// SyncBudget=300ms (Soll): the loop returns inside the budget, the block +// is memoized caller_timeout with next_attempt_at <= now(), the query +// path never picks it again, the background arm's predicate still does. +// +// Run: `go test -tags=integration ./internal/handler/ -run TestBackfillPending_SyncBudget -count=1 -v`. +package handler + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/GottZ/ctx/internal/blocktype" + "github.com/GottZ/ctx/internal/config" + "github.com/GottZ/ctx/internal/store" + "github.com/GottZ/ctx/internal/testdb" +) + +// slowEmbedServer answers like fakeEmbedServer, but only after `delay`, or +// at once with the request's cancellation if the client goes away first. +func slowEmbedServer(t *testing.T, delay time.Duration) (*httptest.Server, *atomic.Int32, *atomic.Int32) { + t.Helper() + var hits, canceled atomic.Int32 + vec := make([]float64, 1024) + for i := range vec { + vec[i] = float64((i % 2) * 2) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + // Drain the body first: net/http starts the background read that + // turns a client disconnect into r.Context() cancellation only once + // the request body has been consumed (registerOnHitEOF in + // readRequest). Without this the handler never sees the cancel. + _, _ = io.Copy(io.Discard, r.Body) + select { + case <-time.After(delay): + case <-r.Context().Done(): + canceled.Add(1) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"embeddings": [][]float64{vec}}) + })) + t.Cleanup(srv.Close) + return srv, &hits, &canceled +} + +func TestBackfillPending_SyncBudget_Integration(t *testing.T) { + if testing.Short() { + t.Skip("integration test") + } + pool := testdb.SetupTestDB(t) + ctx := context.Background() + + const title = "syncbudget-slow-block" + seed := func(t *testing.T) { + t.Helper() + if _, err := pool.Exec(ctx, + `INSERT INTO context_blocks (category, title, content, scope, created_at, updated_at) + VALUES ('learnings', $1, 'body of the sync-budget fixture block', 'private', now(), now())`, title); err != nil { + t.Fatalf("seed block: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM context_embed_failures WHERE block_id IN (SELECT id FROM context_blocks WHERE title = $1)`, title) + _, _ = pool.Exec(ctx, `DELETE FROM context_blocks WHERE title = $1`, title) + }) + } + readMemo := func(t *testing.T) (class string, dueNow bool, attempts int) { + t.Helper() + err := pool.QueryRow(ctx, + `SELECT f.last_class, f.next_attempt_at <= now(), f.attempts + FROM context_embed_failures f JOIN context_blocks cb ON cb.id = f.block_id + WHERE cb.title = $1 AND f.migration_id IS NULL`, title).Scan(&class, &dueNow, &attempts) + if err != nil { + t.Fatalf("read memo: %v", err) + } + return class, dueNow, attempts + } + newHandler := func(t *testing.T, delay time.Duration) (*QueryHandler, *atomic.Int32, *atomic.Int32) { + t.Helper() + srv, hits, canceled := slowEmbedServer(t, delay) + st := &countingStore{} + st.cfg.Store(snapshotTestConfig()) + h := NewQueryHandler(pool, st, embedPool(srv.URL), nil, blocktype.NewRegistry(), snapshotTestAdmitter(t)) + return h, hits, canceled + } + + // Rot: SyncBudget=0 is the historical shape — the loop waits for the + // slow backend however long it takes. Nothing bounds the request's time. + t.Run("red_unbudgeted_waits_for_the_slow_backend", func(t *testing.T) { + seed(t) + h, hits, _ := newHandler(t, 1500*time.Millisecond) + cfg := &config.Config{EmbedBackfill: config.EmbedBackfillConfig{ + SyncCap: 4, SyncBudget: 0, MaxTokens: 1_000_000, BackoffBase: time.Minute, BackoffCap: time.Hour, + }} + + started := time.Now() + got := h.backfillPending(ctx, nil, "private", h.embedAdmission(), cfg) + elapsed := time.Since(started) + if got != 1 { + t.Fatalf("RED premise: backfilled = %d, want 1 (unbudgeted loop embeds the slow block)", got) + } + if elapsed < 1400*time.Millisecond { + t.Fatalf("RED premise: returned after %v, want >= 1.4 s (the loop waited for the backend)", elapsed) + } + if hits.Load() != 1 { + t.Errorf("wire hits = %d, want 1", hits.Load()) + } + t.Logf("RED confirmed: SyncBudget=0 held the request for %v on one slow block", elapsed) + }) + + // Grün: SyncBudget bounds the loop. The slow block is cut at the budget, + // memoized caller_timeout and due for the background arm at once; the + // parent context is untouched, so the question embed keeps its time. + t.Run("budget_cuts_the_slow_block_and_hands_it_to_the_arm", func(t *testing.T) { + seed(t) + h, hits, canceled := newHandler(t, 5*time.Second) + cfg := &config.Config{EmbedBackfill: config.EmbedBackfillConfig{ + SyncCap: 4, SyncBudget: 300 * time.Millisecond, MaxTokens: 1_000_000, BackoffBase: time.Minute, BackoffCap: time.Hour, + }} + reqCtx, cancelReq := context.WithCancel(ctx) + defer cancelReq() + + started := time.Now() + got := h.backfillPending(reqCtx, nil, "private", h.embedAdmission(), cfg) + elapsed := time.Since(started) + if got != 0 { + t.Fatalf("backfilled = %d, want 0 (the slow block must not count)", got) + } + if elapsed > 2*time.Second { + t.Fatalf("returned after %v, want well inside the 5 s backend delay (budget 300 ms)", elapsed) + } + if reqCtx.Err() != nil { + t.Fatalf("request context is dead (%v); the budget must cut only the embed call", reqCtx.Err()) + } + if hits.Load() != 1 { + t.Errorf("wire hits = %d, want 1", hits.Load()) + } + // The backend saw the cancellation: no 7-minute embed keeps burning CPU + // for a caller that is gone (the live task was cancelled at 30 %). + deadline := time.Now().Add(2 * time.Second) + for canceled.Load() == 0 && time.Now().Before(deadline) { + time.Sleep(20 * time.Millisecond) + } + if canceled.Load() != 1 { + t.Errorf("backend saw %d cancellations, want 1", canceled.Load()) + } + + class, dueNow, attempts := readMemo(t) + if class != string(store.EmbedFailureCallerTimeout) { + t.Errorf("memo class = %q, want %q", class, store.EmbedFailureCallerTimeout) + } + if !dueNow { + t.Errorf("memo next_attempt_at is in the future; a caller_timeout must be due for the background arm at once") + } + if attempts != 1 { + t.Errorf("memo attempts = %d, want 1", attempts) + } + + // Pfad A never picks it again, however much budget it has. + cfg.EmbedBackfill.SyncBudget = 10 * time.Second + h2, hits2, _ := newHandler(t, 0) + if again := h2.backfillPending(ctx, nil, "private", h2.embedAdmission(), cfg); again != 0 { + t.Fatalf("second backfillPending embedded %d, want 0 (caller_timeout blocks belong to the background arm)", again) + } + if hits2.Load() != 0 { + t.Errorf("second handler made %d wire calls, want 0", hits2.Load()) + } + + // The background arm's predicate (the shared one) still sees it. + var visibleToArm int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM context_blocks + WHERE embedding IS NULL AND NOT is_archived AND title = $1`+store.EmbedFailureExcludedPredicate, title). + Scan(&visibleToArm); err != nil { + t.Fatalf("arm predicate: %v", err) + } + if visibleToArm != 1 { + t.Errorf("background predicate sees %d block(s), want 1 (caller_timeout is due at once for the arm)", visibleToArm) + } + }) + + // A request that is cancelled mid-embed (client gone) is the same case + // for the block: caller_timeout, not wire — the backend did nothing wrong. + t.Run("request_cancel_is_caller_timeout_not_wire", func(t *testing.T) { + seed(t) + h, _, _ := newHandler(t, 5*time.Second) + cfg := &config.Config{EmbedBackfill: config.EmbedBackfillConfig{ + SyncCap: 4, SyncBudget: 10 * time.Second, MaxTokens: 1_000_000, BackoffBase: time.Minute, BackoffCap: time.Hour, + }} + reqCtx, cancelReq := context.WithCancel(ctx) + go func() { + time.Sleep(200 * time.Millisecond) + cancelReq() + }() + if got := h.backfillPending(reqCtx, nil, "private", h.embedAdmission(), cfg); got != 0 { + t.Fatalf("backfilled = %d, want 0", got) + } + class, dueNow, _ := readMemo(t) + if class != string(store.EmbedFailureCallerTimeout) { + t.Errorf("memo class = %q, want %q", class, store.EmbedFailureCallerTimeout) + } + if !dueNow { + t.Errorf("memo must be due at once for the background arm") + } + }) +} diff --git a/go/internal/handler/query.go b/go/internal/handler/query.go index 3ab3d61d..db519a5c 100644 --- a/go/internal/handler/query.go +++ b/go/internal/handler/query.go @@ -729,7 +729,10 @@ func (h *QueryHandler) HandleQuery(w http.ResponseWriter, r *http.Request) { "request_id", requestID, ) } else if llm.DetectGerman(query) { - slog.Info("german detected, translating", + // The heuristic fires on umlauts and German stopwords, so Swedish, + // Danish and Finnish questions land here too; the translation to + // English is right for them as well, only the old label was not. + slog.Info("non-english heuristic matched, translating", "request_id", requestID, ) translatedQuery, err := llm.TranslateQuery(ctx, h.pool, h.backendPool, querySens, query, ar.ApiKeyID, h.admission()) @@ -1710,12 +1713,34 @@ func (h *QueryHandler) logAccess(ar *auth.AuthResult, results []rrf.SearchResult func (h *QueryHandler) backfillPending(ctx context.Context, floor config.ScopeFloor, scope string, adm embedcache.Admission, cfg *config.Config) int { count := 0 syncCap := cfg.EmbedBackfill.SyncCap + // Time budget next to the count cap (embed_backfill.sync_budget, issue + // damienmoon/ctx#1): SyncCap bounds how MANY blocks a request embeds + // inline, this bounds how LONG. One block that cannot embed inside the + // caller's deadline used to consume the whole deadline — the question's + // own embed was then never admitted and the query answered 500. Every + // block's wire call now runs under what is left of the budget; a block + // that overruns it is memoized caller_timeout and left to the + // background arm (runEmbedBackfillLoop), and the loop ends so the + // question embed keeps the rest of the request. Zero = unbounded, the + // pre-budget shape (explicit opt-out, same convention as SyncCap). + var budgetDeadline time.Time + if budget := cfg.EmbedBackfill.SyncBudget; budget > 0 { + budgetDeadline = time.Now().Add(budget) + } for attempts := 0; syncCap <= 0 || attempts < syncCap; attempts++ { + if !budgetDeadline.IsZero() && !time.Now().Before(budgetDeadline) { + slog.Info("query backfill: sync budget spent, rest left to the background arm", + "budget", cfg.EmbedBackfill.SyncBudget, "backfilled", count) + break + } var blockID, title, content, sens, scope string + // Pfad A's predicate: the shared backoff exclusion PLUS every block + // already memoized caller_timeout (store.EmbedFailureQueryPath + // ExcludedPredicate) — such a block is the background arm's job. err := h.pool.QueryRow(ctx, `SELECT id, title, content, sensitivity, scope FROM context_blocks WHERE embedding IS NULL AND NOT is_archived`+ - store.EmbedFailureExcludedPredicate+store.RetrievalExcludedTypePredicate+` + store.EmbedFailureQueryPathExcludedPredicate+store.RetrievalExcludedTypePredicate+` LIMIT 1`).Scan(&blockID, &title, &content, &sens, &scope) if err != nil { break // No more pending blocks (or error). @@ -1758,9 +1783,17 @@ func (h *QueryHandler) backfillPending(ctx context.Context, floor config.ScopeFl // pool=nil: document embeddings land in the block row, not the cache // (today's semantics — the cache is for repeated query/keyword text). + // The wire call runs under the remaining sync budget; callerCut is + // read BEFORE cancelEmbed, which would make Err() non-nil by itself. + embedCtx, cancelEmbed := ctx, func() {} + if !budgetDeadline.IsZero() { + embedCtx, cancelEmbed = context.WithDeadline(ctx, budgetDeadline) + } vec, served, wireAttempts, wired, err := embedcache.EmbedChain( - ctx, nil, chain, backends.RoleEmbed, embedText, embed.PrefixDocument, + embedCtx, nil, chain, backends.RoleEmbed, embedText, embed.PrefixDocument, embedcache.ReportFunc(llm.PoolReporter(h.backendPool)), adm) + callerCut := err != nil && embedCtx.Err() != nil + cancelEmbed() if wired { // TENANT-DECISION(backfill-attribution): "" → NULL. backfillPending is // query-triggered but maintenance in nature — it embeds whatever blocks @@ -1782,7 +1815,20 @@ func (h *QueryHandler) backfillPending(ctx context.Context, floor config.ScopeFl // failure most often means the backend itself is unavailable, // and hammering it for the rest of the sync-cap budget within // one request helps nobody. - class, normalized := store.ClassifyEmbedError(err) + // A budget or request deadline is a statement about THIS caller, + // not about the backend: class caller_timeout, which the query + // path never re-picks and the background arm picks at once. + // Everything else keeps the wire/oversize classification. + var ( + class store.EmbedFailureClass + normalized string + ) + if callerCut { + class = store.EmbedFailureCallerTimeout + normalized = store.NormalizeEmbedError(class, err.Error()) + } else { + class, normalized = store.ClassifyEmbedError(err) + } // The memo write must survive the request context: the wire // failure above IS frequently the request deadline itself // (canceled context), and a memo lost to that same cancellation diff --git a/go/internal/store/embed_failures.go b/go/internal/store/embed_failures.go index 4d54d09f..f4f3a0cc 100644 --- a/go/internal/store/embed_failures.go +++ b/go/internal/store/embed_failures.go @@ -44,6 +44,17 @@ const ( // memo-free "stays unembedded, retried every cycle" semantics for this // case — a pool outage there must not permanently park blocks). EmbedFailureSensitivityIneligible EmbedFailureClass = "sensitivity_ineligible" + // EmbedFailureCallerTimeout marks a block whose embed did not fit the + // query path's sync budget (embed_backfill.sync_budget) or died with + // the request context — the backend is not known to be down, it was + // slow relative to THIS caller's deadline (issue damienmoon/ctx#1: a + // 10k-token block on a CPU backend). The memo therefore says nothing + // about the backend and must not park the block on the wire curve: the + // query path (EmbedFailureQueryPathExcludedPredicate) never picks a + // caller_timeout block again, the background arm picks it at once + // (next_attempt_at = now()). A later wire failure in the background + // arm reclassifies it as wire with the ordinary backoff. + EmbedFailureCallerTimeout EmbedFailureClass = "caller_timeout" ) // maxLastErrorLen mirrors migration 113's documented last_error contract: @@ -153,6 +164,27 @@ func RecordEmbedFailure(ctx context.Context, q pgxdb.Execer, blockID string, cla return nil } + if class == EmbedFailureCallerTimeout { + // Eligible for the background arm at once: the backoff curve is + // for a backend that failed, this backend only took longer than + // the caller could wait. Pfad A's own exclusion of this class is + // in the predicate, not in the timestamp. + _, err := q.Exec(ctx, + `INSERT INTO context_embed_failures (block_id, migration_id, attempts, last_error, last_class, next_attempt_at) + VALUES ($1, NULL, 1, $2, $3, now()) + ON CONFLICT (block_id) WHERE migration_id IS NULL + DO UPDATE SET attempts = context_embed_failures.attempts + 1, + last_error = EXCLUDED.last_error, + last_class = EXCLUDED.last_class, + next_attempt_at = now()`, + blockID, normalizedErr, string(class), + ) + if err != nil { + return fmt.Errorf("store: record embed failure (caller_timeout): %w", err) + } + return nil + } + baseSecs := backoffBase.Seconds() capSecs := backoffCap.Seconds() _, err := q.Exec(ctx, @@ -265,3 +297,21 @@ const EmbedFailureExcludedPredicate = ` AND f.migration_id IS NULL AND f.next_attempt_at > now() )` + +// EmbedFailureQueryPathExcludedPredicate is Pfad A's (query.go +// backfillPending) variant of EmbedFailureExcludedPredicate: the same +// backoff exclusion PLUS every block memoized as caller_timeout, for good. +// A block that once overran the sync budget is the background arm's job +// from then on; re-picking it inline would burn the budget of every later +// query for a block the arm is about to embed anyway (and, while the arm +// holds its FOR UPDATE row, embed it a second time in parallel). The +// background arm keeps the plain predicate, so for it a caller_timeout memo +// (next_attempt_at = now()) is immediately eligible. Same positional +// contract as its sibling: bare context_blocks as the outer FROM. +const EmbedFailureQueryPathExcludedPredicate = ` + AND NOT EXISTS ( + SELECT 1 FROM context_embed_failures f + WHERE f.block_id = context_blocks.id + AND f.migration_id IS NULL + AND (f.next_attempt_at > now() OR f.last_class = 'caller_timeout') + )` From 83a0aca304094d9cc9947190f48b478e7badb771 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Fri, 9 Oct 2026 04:19:27 +0200 Subject: [PATCH 02/16] docs(backfill): name the tracking issue by its URL The comments and the operations paragraph said "issue damienmoon/ctx#1", which on GitHub reads as a reference into a repository that does not exist there. The issue lives on the author's Forgejo, a read-only mirror of the GitHub fork: https://force-push.me/damienmoon/ctx/issues/1. Comments only; no code changes. Co-Authored-By: Claude Fable 5.1 --- docs/operations.md | 2 +- go/internal/config/config.go | 4 ++-- go/internal/events/backfill_arm_integration_test.go | 2 +- go/internal/events/scheduler.go | 4 ++-- go/internal/handler/backfill_syncbudget_integration_test.go | 2 +- go/internal/handler/query.go | 2 +- go/internal/store/embed_failures.go | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index b57b1a7b..c78180bb 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -477,7 +477,7 @@ The `llama-embed` sidecar (`docker-compose.override.yml`, Qwen3-Embedding-8B, `- **Backfill failure memo (migration 113, Achse 04 W04-2).** Both backfill paths — the scheduler arm (`backfillOneEmbedding`) and the query-path pre-search loop (`backfillPending`) — used to retry a NULL-embedding block **indefinitely** if it exceeded the embed window: no truncate, no skip, nothing written to the DB on failure, so the next pick landed on the exact same block (observed 2026-07-10: a 13k-char block at 4704 tokens against a 4096 window looped every ~20 s and blocked ~265 pending blocks behind it). This is closed structurally, not operationally: every failed embed attempt now upserts a row into `context_embed_failures` (`block_id`, `attempts`, `last_error` — normalized, ≤500 chars, never raw wire/embed-text content —, `last_class`, `next_attempt_at`), and both pick queries exclude a block whose `next_attempt_at` is still in the future. A non-oversize failure (backend down, transient wire error) gets an exponential backoff (`embed_backfill.backoff_base` × 2^(attempts-1), capped at `embed_backfill.backoff_cap`; defaults 1min/24h; the base must be > 0 — V21 rejects 0, which would mean "retry immediately" and a tight loop against a failing backend, issue #38) — the next PEEK reaches the next-oldest block instead of re-picking the same one. An oversize block — classified either by a conservative pre-wire token estimate (`len(text)/4` against `embed_backfill.max_tokens`, default 24000, chosen to sit under the live kv-unified pool's 32k minus interactive headroom — see the tuning section above) or by an HTTP 400 whose body names `exceed_context_size` — gets `next_attempt_at = 'infinity'` instead: permanently parked, never a blind 24h-in-slow-motion retry, but never silently dropped either (the memo row stays visible in `context_embed_failures`). Known limitation: `ClearEmbedding` does NOT currently clear the block's failure memo, so a content edit that shrinks a previously oversize-parked block below the token threshold does **not** reactivate it on its own — the `infinity` exclusion persists until the memo row is cleaned up (a janitor for this is not yet built; see the Achse-04 backlog). The query-path loop additionally carries a synchronous cap (`embed_backfill.sync_cap`, default 4): before this wave a large rest-transient (e.g. after a bulk ingest that outran the scheduler arm) made the *first* interactive query pay the entire nachzug inline; now it embeds at most `sync_cap` blocks per request and leaves the rest to the scheduler arm. Keeping blocks compact (~1–1.5k chars per the store guidance) or sizing the embed window above your largest block is still the actual fix for an oversize block's content — the memo mechanism only stops it from starving everything behind it. -**The dedicated backfill arm and the query-path time budget (issue damienmoon/ctx#1).** Two gaps in the above showed up on a CPU embed backend (Qwen3-Embedding-8B at the documented ~23 tok/s) with a ~10k-token block. First, the scheduler arm ran only at the top of the dream loop, so with `dream.enabled=false` — the default — nothing backfilled pending embeddings in the background at all; the query path was the only path. `runEmbedBackfillLoop` is now its own goroutine, started regardless of Dream: a successful pick loops at once, an empty pick waits `embed_backfill.interval` (default 15 s), `0` turns the arm off (polled every 15 s for a hot re-enable). The dream loop keeps its inline call; the tx-wrapped `FOR UPDATE SKIP LOCKED` pick makes two pickers safe. Second, `sync_cap` bounds how *many* blocks a query embeds inline, not how *long*: one block that needs 7 minutes consumed the caller's whole 120 s deadline, the question's own embed was never admitted, and the query answered 500 instead of degrading. `embed_backfill.sync_budget` (seconds, default 30, `0` = unbounded) now bounds the whole pre-search loop per request; each wire call runs under what is left of it. A block that overruns the budget — or dies with the request context — is memoized with the class `caller_timeout`: the query path never picks it again (`EmbedFailureQueryPathExcludedPredicate`), the background arm picks it at once (`next_attempt_at = now()`), and a later wire failure there reclassifies it as `wire` with the ordinary backoff. The block stays FTS-visible meanwhile. Previously such a failure was memoized as `wire` and re-burned the deadline of every query that landed after the backoff lapsed. +**The dedicated backfill arm and the query-path time budget (tracking issue: force-push.me/damienmoon/ctx/issues/1).** Two gaps in the above showed up on a CPU embed backend (Qwen3-Embedding-8B at the documented ~23 tok/s) with a ~10k-token block. First, the scheduler arm ran only at the top of the dream loop, so with `dream.enabled=false` — the default — nothing backfilled pending embeddings in the background at all; the query path was the only path. `runEmbedBackfillLoop` is now its own goroutine, started regardless of Dream: a successful pick loops at once, an empty pick waits `embed_backfill.interval` (default 15 s), `0` turns the arm off (polled every 15 s for a hot re-enable). The dream loop keeps its inline call; the tx-wrapped `FOR UPDATE SKIP LOCKED` pick makes two pickers safe. Second, `sync_cap` bounds how *many* blocks a query embeds inline, not how *long*: one block that needs 7 minutes consumed the caller's whole 120 s deadline, the question's own embed was never admitted, and the query answered 500 instead of degrading. `embed_backfill.sync_budget` (seconds, default 30, `0` = unbounded) now bounds the whole pre-search loop per request; each wire call runs under what is left of it. A block that overruns the budget — or dies with the request context — is memoized with the class `caller_timeout`: the query path never picks it again (`EmbedFailureQueryPathExcludedPredicate`), the background arm picks it at once (`next_attempt_at = now()`), and a later wire failure there reclassifies it as `wire` with the ordinary backoff. The block stays FTS-visible meanwhile. Previously such a failure was memoized as `wire` and re-burned the deadline of every query that landed after the backoff lapsed. **Backfill skips retrieval-excluded types (2026-07-25).** Both backfill paths now also skip a block whose *type* carries retrieval policy `excluded` in the block-type registry (shipped builtins: `checkpoint`, `system-meta`). Such a vector has no consumer at all — every ranked path filters against the registry's visible-type allowlist, so the embedding was written once and then read by nobody. The trigger was a live incident on 2026-07-25 with two independent arms: 33 Hermes "Compaction source" parts (`type=checkpoint`, ~36.5 kB each) saturated the CPU embed backend for hours, and the digest rebuild rewrites `topic-map-private` (`type=system-meta`, 73.7 kB ≈ ~32k real tokens) on **every** boot, which clears its embedding and bought ~60 min of CPU embed for a dead vector — recurring, forever. The filter is policy-driven, not a hardcoded type list (same mechanic as `test.sh` T19), and deliberately conservative in the fail-safe direction: only `_global`-`excluded` skips; `damped` and `aggregate-to-parent` stay embeddable, a block whose type has no registry row at all stays embeddable, and a tenant scope that overrides an excluded type to a visible policy re-opens that type for the backfill (the pick has no reader identity, so the reader-aware overlay rule is mirrored as the union over all possible readers). Two operational consequences: blocks of an excluded type keep `embedding IS NULL` forever — that is the intended end state, not a stuck queue — and the `/status` `embed_backlog` metric still counts them, so a corpus with such blocks shows a permanent backlog floor (live at the time of writing: 24 `checkpoint` + 1 `system-meta`). Existing `infinity` failure memos for such blocks become moot: the predicate no longer even sees the rows. Flipping a type's policy from `excluded` to a visible one makes its blocks embeddable again on the next cycle without any manual step. diff --git a/go/internal/config/config.go b/go/internal/config/config.go index 71cf7a69..af0f5f96 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -1596,7 +1596,7 @@ type EmbedBackfillConfig struct { // whole pre-search loop of one request runs inside this wall-clock // budget, and each block's embed call runs under what is left of it. // SyncCap alone could not stop one slow block from eating the caller's - // entire deadline (issue damienmoon/ctx#1: a ~10k-token block against a + // entire deadline (the tracking issue, force-push.me/damienmoon/ctx/issues/1: a ~10k-token block against a // CPU embed backend at ~23 tok/s needs ~7 min, the CLI client's deadline // is 120 s — the question's own embed was never admitted and the query // answered 500 instead of degrading). A block that overruns the budget @@ -1610,7 +1610,7 @@ type EmbedBackfillConfig struct { // this key the only background backfill was the call at the top of the // dream loop, so with dream.enabled=false (the default) NOTHING embedded // pending blocks in the background and Pfad A was the only path (issue - // damienmoon/ctx#1). The arm runs regardless of Dream; a successful + // force-push.me/damienmoon/ctx/issues/1). The arm runs regardless of Dream; a successful // pick loops at once, an empty pick waits this long. 0 = arm off // (polled every 15 s for a hot re-enable, contract.recheck_interval // convention). diff --git a/go/internal/events/backfill_arm_integration_test.go b/go/internal/events/backfill_arm_integration_test.go index 9c223a12..55383e6c 100644 --- a/go/internal/events/backfill_arm_integration_test.go +++ b/go/internal/events/backfill_arm_integration_test.go @@ -1,6 +1,6 @@ //go:build integration -// Rot-Gate for issue damienmoon/ctx#1, scheduler half: the embed-backfill +// Rot-Gate for the tracking issue, force-push.me/damienmoon/ctx/issues/1, scheduler half: the embed-backfill // arm (backfillOneEmbedding) was called ONLY at the top of runDreamLoop, so // with dream.enabled=false — the registry default — no goroutine ever // embedded a pending block in the background. Live: a block stayed pending diff --git a/go/internal/events/scheduler.go b/go/internal/events/scheduler.go index 8edbfc78..20f754e5 100644 --- a/go/internal/events/scheduler.go +++ b/go/internal/events/scheduler.go @@ -782,7 +782,7 @@ func (s *Scheduler) Run(ctx context.Context) { embedMigrateTicker := time.NewTicker(embedMigrateInterval) defer embedMigrateTicker.Stop() - // The dedicated embed-backfill arm (issue damienmoon/ctx#1): pending + // The dedicated embed-backfill arm (tracking issue: force-push.me/damienmoon/ctx/issues/1): pending // embeddings used to be backfilled only at the top of the dream loop, so // with dream.enabled=false (the default) or Dream switched off at runtime // nothing embedded them in the background. The arm runs regardless of @@ -2057,7 +2057,7 @@ func (t *txTail) done() { const backfillArmPoll = 15 * time.Second // runEmbedBackfillLoop is the dedicated background arm for pending -// embeddings (issue damienmoon/ctx#1). It is the same work as the call at the +// embeddings (tracking issue: force-push.me/damienmoon/ctx/issues/1). It is the same work as the call at the // top of runDreamLoop — backfillOneEmbedding under the iterated tenant's // snapshot and router — on its own cadence: a successful pick loops at once, // an empty pick waits embed_backfill.interval, an error waits 10 s. Under diff --git a/go/internal/handler/backfill_syncbudget_integration_test.go b/go/internal/handler/backfill_syncbudget_integration_test.go index 077115ce..bcfba1b2 100644 --- a/go/internal/handler/backfill_syncbudget_integration_test.go +++ b/go/internal/handler/backfill_syncbudget_integration_test.go @@ -1,6 +1,6 @@ //go:build integration -// Rot-Gate for issue damienmoon/ctx#1: the query-path pre-search backfill +// Rot-Gate for the tracking issue, force-push.me/damienmoon/ctx/issues/1: the query-path pre-search backfill // (backfillPending) had a COUNT cap (embed_backfill.sync_cap) but no TIME // budget. One block whose embed could not finish inside the caller's // deadline consumed the entire deadline (live: a ~10k-token block against a diff --git a/go/internal/handler/query.go b/go/internal/handler/query.go index db519a5c..e18abbb5 100644 --- a/go/internal/handler/query.go +++ b/go/internal/handler/query.go @@ -1714,7 +1714,7 @@ func (h *QueryHandler) backfillPending(ctx context.Context, floor config.ScopeFl count := 0 syncCap := cfg.EmbedBackfill.SyncCap // Time budget next to the count cap (embed_backfill.sync_budget, issue - // damienmoon/ctx#1): SyncCap bounds how MANY blocks a request embeds + // force-push.me/damienmoon/ctx/issues/1): SyncCap bounds how MANY blocks a request embeds // inline, this bounds how LONG. One block that cannot embed inside the // caller's deadline used to consume the whole deadline — the question's // own embed was then never admitted and the query answered 500. Every diff --git a/go/internal/store/embed_failures.go b/go/internal/store/embed_failures.go index f4f3a0cc..2c111372 100644 --- a/go/internal/store/embed_failures.go +++ b/go/internal/store/embed_failures.go @@ -47,7 +47,7 @@ const ( // EmbedFailureCallerTimeout marks a block whose embed did not fit the // query path's sync budget (embed_backfill.sync_budget) or died with // the request context — the backend is not known to be down, it was - // slow relative to THIS caller's deadline (issue damienmoon/ctx#1: a + // slow relative to THIS caller's deadline (the tracking issue, force-push.me/damienmoon/ctx/issues/1: a // 10k-token block on a CPU backend). The memo therefore says nothing // about the backend and must not park the block on the wire curve: the // query path (EmbedFailureQueryPathExcludedPredicate) never picks a From 218d5f84949ad055bbe18599a4fb8322c746d7c5 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Fri, 9 Oct 2026 04:55:02 +0200 Subject: [PATCH 03/16] test(config): raise the duration-key drift guard to 47 embed_backfill.sync_budget and embed_backfill.interval are two new typDuration keys. TestValidateRejectsNegativeOnEveryDurationKey pins the count as a drift guard (45 before this branch) and failed in CI. Both keys are covered by the generic V17 sign walk, which the test exercises per key; the pin moves to 47 in the commit that explains why, as the comment above it asks. Co-Authored-By: Claude Fable 5.1 --- go/internal/config/validate_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/internal/config/validate_test.go b/go/internal/config/validate_test.go index 44604181..d027bb74 100644 --- a/go/internal/config/validate_test.go +++ b/go/internal/config/validate_test.go @@ -393,7 +393,7 @@ func TestV17bIsAdvisoryOnly(t *testing.T) { // changes TYPE (seconds → int, or a struct field that stops being a // time.Duration) would leave the walk without anyone noticing. Raise this // number in the same commit that adds a duration key. -const wantDurationKeys = 45 +const wantDurationKeys = 47 // +2 on 2026-10-09: embed_backfill.sync_budget and embed_backfill.interval, both V17-covered // TestValidateRejectsNegativeOnEveryDurationKey is the registry-wide form of // V17 (issue #29): before it, exactly two of the seconds keys had a sign check From 89eb8c789f4014d3e2d55098534bd99b760e34f2 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 00:06:06 +0200 Subject: [PATCH 04/16] feat(backends): anthropic wire protocol constant, validation and Migration 155 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fourth wire protocol, "anthropic", for the native Anthropic Messages API (POST /v1/messages). This commit adds the constant and the row shape the wire needs; the wire itself follows in go/internal/llm. Validation (422 on create and update, protocol-bound so openai/ollama rows keep their freedoms): api_key_ref required (the key travels as x-api-key), locality external, num_ctx > 0 (no window discovery on this wire — synthesis would fail closed on promptguard.ErrUndeclaredWindow), provider_class generic, no embed/dream-embed/rerank roles (the Messages API has no such endpoints; embed.go would silently dispatch them onto the Ollama path), effort in extra_body or model_map params limited to low|medium|high|xhigh|max, and the /v1 root guard extended (the wire appends /v1/messages itself). Migration 155 widens the CHECK on context_backends.protocol. The baseline declared it inline and unnamed, so the DO block resolves the name from pg_constraint by table and definition (the pattern 113 and 114 already use) before re-adding it as context_backends_protocol_check. Manifest: the schema contract records column types, indexes, functions, triggers, rules and hypertables — no CHECK clauses — so a regeneration after 155 changes only generated_against.migration_max (155) and migration_count (43). Those two fields are bumped by hand here because the generator needs Docker (testcontainers), which the authoring seat does not have; the integration job's contract check verifies the manifest against the live chain. Tests: TestValidateAnthropicHappyPath, TestValidateAnthropicRules (one mutation per rule, plus the valid effort set and the protocol-boundedness check). docs/architecture.md names the protocol. Co-Authored-By: Claude Fable 5.1 --- docs/architecture.md | 4 +- go/internal/backends/backend.go | 28 ++++++- go/internal/backends/validate.go | 58 +++++++++++++- go/internal/backends/validate_test.go | 80 +++++++++++++++++++ go/internal/schemacontract/manifest.json | 4 +- .../155_backend_protocol_anthropic.sql | 41 ++++++++++ 6 files changed, 207 insertions(+), 8 deletions(-) create mode 100644 go/migrations/155_backend_protocol_anthropic.sql diff --git a/docs/architecture.md b/docs/architecture.md index 8eaf7242..6aa61c57 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -**Stack:** Go 1.27 (matches `go.mod` and the `golang:1.27-alpine` build image), PostgreSQL 18 + pgvector 0.8.2, 84 SQL migrations (numbered 001–086; 066 is the deliberately deferred tenant-OAuth slot, 082 is the reserved W13 webhook slot). Dual-protocol inference (Ollama native or OpenAI-compatible) via any provider — configured per role in the [backend pool](#backend-pool-f3-migrations-053055) (`ctx backends`), one row per backend (see [operations](operations.md#backends)). +**Stack:** Go 1.27 (matches `go.mod` and the `golang:1.27-alpine` build image), PostgreSQL 18 + pgvector 0.8.2, 84 SQL migrations (numbered 001–086; 066 is the deliberately deferred tenant-OAuth slot, 082 is the reserved W13 webhook slot). Multi-protocol inference (Ollama native, OpenAI-compatible, or the native Anthropic Messages API) via any provider — configured per role in the [backend pool](#backend-pool-f3-migrations-053055) (`ctx backends`), one row per backend (see [operations](operations.md#backends)). ## Pipelines at a glance @@ -372,7 +372,7 @@ Since T10 the registry is **API-exposed** (no schema change) via the `type-*` ma ## Backend pool (F3, migrations 053–055) -`context_backends` replaces the hardwired primary+fallback pair with a declarative, role-routed, priority-ordered pool. Each row is one backend: `base_url`, wire `protocol` (`openai`/`ollama`/`rerank`), `provider_class` (`generic`/`llamacpp`/`openrouter`), a **trust level**, an egress `locality`, a `roles` list (`synthesis`, `translate`, `embed`, `rerank`, `dream`, `digest`, `chat`, `classify`, free-form), a per-role `model_map`, per-role `timeouts`, `priority` and `enabled`. Order/priority are pure DATA — no code path references backend names or priority constants. +`context_backends` replaces the hardwired primary+fallback pair with a declarative, role-routed, priority-ordered pool. Each row is one backend: `base_url`, wire `protocol` (`openai`/`ollama`/`anthropic`/`rerank` — `anthropic` is the native Messages API wire, chat-class roles only, see [api — Anthropic](api.md#backend-pool)), `provider_class` (`generic`/`llamacpp`/`openrouter`), a **trust level**, an egress `locality`, a `roles` list (`synthesis`, `translate`, `embed`, `rerank`, `dream`, `digest`, `chat`, `classify`, free-form), a per-role `model_map`, per-role `timeouts`, `priority` and `enabled`. Order/priority are pure DATA — no code path references backend names or priority constants. `context_disable_profiles` + `context_disable_profile_backends` (092) add named **disable profiles** — a profile takes its member backends out of every chain while active (maintenance / eject use-case). Membership is a real FK join (no name matching, rename/delete-safe by CASCADE), profiles carry a `scope` (`_global` for operator profiles) with `UNIQUE(scope, name)`, and the migration backfills the legacy gaming toggle as the reserved profile `eject` (label „Eject-Modus“) — value and member set copied losslessly from `gaming.active`/`gaming.disabled_backends`. The pool loads profiles + memberships into the same atomic snapshot as the backends (ORDER BY name — the status frame's diffKey needs a byte-stable order). The chain-time exclusion arm is live: an active profile takes its members out of every chain (`disabled by profile `, effective state `profile-disabled`, precedence disabled > profile-disabled > cooldown > active). Since U01-W5 the eject profile is the **sole** chain-time exclusion mechanism — the legacy `gaming.active`/`gaming.disabled_backends` settings keys and the parallel gaming arm were retired (the `gaming-mode`→`eject` alias shim stays as a compat surface, and the status payload's `gaming.active` field is derived live from the eject profile). diff --git a/go/internal/backends/backend.go b/go/internal/backends/backend.go index e9030b71..2aa44860 100644 --- a/go/internal/backends/backend.go +++ b/go/internal/backends/backend.go @@ -79,11 +79,35 @@ type Protocol string // Supported wire protocols. Empty is only valid where a field documents // inheritance (dream-embed); config validation V4 rejects everything else. +// +// ProtocolAnthropic is the native Anthropic Messages API (POST /v1/messages, +// x-api-key + anthropic-version headers). It is a chat-only wire: the +// Messages API serves no embeddings and no rerank, so validation keeps the +// embed/dream-embed/rerank roles off such a row. Unlike the openai wire it +// carries NO sampling parameters (the Claude 5 models answer 400 to +// temperature/top_p/top_k), caches the system prompt (cache_control +// ephemeral) and takes an effort level (output_config.effort) instead of a +// think toggle — see llm/anthropic.go. const ( - ProtocolOllama Protocol = "ollama" - ProtocolOpenAI Protocol = "openai" + ProtocolOllama Protocol = "ollama" + ProtocolOpenAI Protocol = "openai" + ProtocolAnthropic Protocol = "anthropic" ) +// AnthropicEfforts are the accepted output_config.effort values of the +// anthropic wire (claude-api reference, 2026-10). Validation rejects any +// other extra_body.effort / model_map params.effort at write time so a typo +// never reaches the provider as a 400 on every call. +var AnthropicEfforts = map[string]bool{ + "low": true, "medium": true, "high": true, "xhigh": true, "max": true, +} + +// AnthropicDefaultEffort is sent when neither the role's model_map params nor +// the row's extra_body name an effort. It matches the model-side default of +// Claude Opus 5.5 (medium) and is sent EXPLICITLY so the effective level is +// visible in every request body rather than implied by the model. +const AnthropicDefaultEffort = "medium" + // ThinkMode is the reasoning toggle as configured: "true" | "false" | "" // (= omit from the request). type ThinkMode string diff --git a/go/internal/backends/validate.go b/go/internal/backends/validate.go index b115f304..f8e1e964 100644 --- a/go/internal/backends/validate.go +++ b/go/internal/backends/validate.go @@ -118,6 +118,7 @@ func ValidateBackend(b *Backend) (warnings []string, errs []FieldError) { warnings = append(warnings, w...) errs = append(errs, e...) errs = append(errs, validateCarriers(b)...) + errs = append(errs, validateAnthropic(b)...) return warnings, errs } @@ -148,9 +149,9 @@ func validateIdentity(b *Backend) (errs []FieldError) { errs = append(errs, validateVersionedRoot(b)...) switch b.Protocol { - case ProtocolOpenAI, ProtocolOllama, "rerank": + case ProtocolOpenAI, ProtocolOllama, ProtocolAnthropic, "rerank": default: - errs = append(errs, FieldError{"protocol", "must be one of: openai, ollama, rerank"}) + errs = append(errs, FieldError{"protocol", "must be one of: openai, ollama, anthropic, rerank"}) } switch b.ProviderClass { @@ -221,10 +222,63 @@ func validateVersionedRoot(b *Backend) []FieldError { case b.Protocol == ProtocolOpenAI || b.Protocol == "rerank": return []FieldError{{"base_url", "base_url is the API root WITHOUT the version segment (the openai/rerank wire paths append /v1/... themselves) — a /v1 root doubles to /v1/v1/... and 404s chat, embeddings and rerank"}} + case b.Protocol == ProtocolAnthropic: + // Same doubling on the Messages API: the wire path is + // {base_url}/v1/messages, the probe {base_url}/v1/models. + return []FieldError{{"base_url", + "base_url is the API root WITHOUT the version segment (the anthropic wire appends /v1/messages itself) — use https://api.anthropic.com, not .../v1"}} } return nil } +// validateAnthropic covers the row shape the native Messages API wire needs +// (llm/anthropic.go). Every rule here is a runtime failure turned into a 422: +// a keyless row 401s on the first call, a non-external locality would drop +// api.anthropic.com from the egress audit, a missing num_ctx fails synthesis +// closed (promptguard.ErrUndeclaredWindow — there is no window discovery for +// this wire), a non-generic provider class would run the OpenRouter ZDR force +// or the llama.cpp /health probe against a provider that has neither, and an +// embed/rerank role would fall onto the Ollama /api/embed path (embed.go +// dispatches "anything but openai" there) with a vector-space corruption +// risk — the Messages API has no embeddings endpoint at all. +func validateAnthropic(b *Backend) (errs []FieldError) { + if b.Protocol != ProtocolAnthropic { + return nil + } + if strings.TrimSpace(b.APIKeyRef) == "" { + errs = append(errs, FieldError{"api_key_ref", "anthropic protocol requires api_key_ref (create the secret first; the key travels as x-api-key)"}) + } + if b.Locality != "" && b.Locality != LocalityExternal { + errs = append(errs, FieldError{"locality", "anthropic protocol requires locality 'external' (the Messages API is a hosted provider)"}) + } + if b.NumCtx <= 0 { + errs = append(errs, FieldError{"num_ctx", "anthropic protocol requires num_ctx > 0 (no window discovery on this wire; synthesis fails closed without a declared window)"}) + } + if b.ProviderClass != "" && b.ProviderClass != ProviderGeneric { + errs = append(errs, FieldError{"provider_class", "anthropic protocol requires provider_class 'generic'"}) + } + for _, r := range b.Roles { + if embedRoles[r] || r == RoleRerank { + errs = append(errs, FieldError{"roles", + fmt.Sprintf("role %q is not served by the anthropic protocol (the Messages API has no embeddings or rerank endpoint)", r)}) + } + } + if raw, ok := b.ExtraBody["effort"]; ok { + if v, isStr := raw.(string); !isStr || !AnthropicEfforts[v] { + errs = append(errs, FieldError{"extra_body", "effort must be one of: low, medium, high, xhigh, max"}) + } + } + for role, spec := range b.ModelMap { + if raw, ok := spec.Params["effort"]; ok { + if v, isStr := raw.(string); !isStr || !AnthropicEfforts[v] { + errs = append(errs, FieldError{"model_map", + fmt.Sprintf("role %q: params.effort must be one of: low, medium, high, xhigh, max", role)}) + } + } + } + return errs +} + // validateRoles covers role/model coverage, the embed+external hard block // and the dream-on-local warning. func validateRoles(b *Backend) (warnings []string, errs []FieldError) { diff --git a/go/internal/backends/validate_test.go b/go/internal/backends/validate_test.go index b4a28ffc..e7ccd22a 100644 --- a/go/internal/backends/validate_test.go +++ b/go/internal/backends/validate_test.go @@ -434,3 +434,83 @@ func TestScoreDomainResolution(t *testing.T) { t.Errorf("invalid value: ScoreDomain() = %q, want auto fallback", got) } } + +// anthropicBackend is the minimal row the native Messages API wire accepts: +// keyed, external, generic, with a declared window and a chat-class role. +func anthropicBackend() Backend { + b := validBackend() + b.Protocol = ProtocolAnthropic + b.Host = "https://api.anthropic.com" + b.APIKeyRef = "anthropic-chat" + b.NumCtx = 100000 + b.Roles = []string{RoleSynthesis, RoleTranslate, RoleDream, RoleDigest, RoleChat} + return b +} + +// TestValidateAnthropicHappyPath pins that the documented minimal row passes +// — every rule test below mutates exactly one field away from it, so a +// regression in the fixture itself would turn all of them green for the +// wrong reason. +func TestValidateAnthropicHappyPath(t *testing.T) { + b := anthropicBackend() + if _, errs := ValidateBackend(&b); len(errs) != 0 { + t.Fatalf("minimal anthropic row rejected: %v", errs) + } +} + +// TestValidateAnthropicRules walks the runtime failures the protocol turns +// into 422s: keyless (401 on first call), non-external locality (egress audit +// hole), missing num_ctx (synthesis fails closed, no window discovery), +// non-generic class (ZDR force / llama.cpp probe against the wrong provider), +// embed and rerank roles (no such endpoints — embed.go would silently take +// the Ollama path), and a mistyped effort in either carrier. +func TestValidateAnthropicRules(t *testing.T) { + cases := []struct { + name string + mut func(b *Backend) + field string + }{ + {"keyless", func(b *Backend) { b.APIKeyRef = "" }, "api_key_ref"}, + {"lan locality", func(b *Backend) { b.Host = "http://10.0.0.2:8080"; b.Locality = LocalityLAN }, "locality"}, + {"no num_ctx", func(b *Backend) { b.NumCtx = 0 }, "num_ctx"}, + {"openrouter class", func(b *Backend) { b.ProviderClass = ProviderOpenRouter }, "provider_class"}, + {"llamacpp class", func(b *Backend) { b.ProviderClass = ProviderLlamaCpp }, "provider_class"}, + {"embed role", func(b *Backend) { b.Roles = append(b.Roles, RoleEmbed) }, "roles"}, + {"dream-embed role", func(b *Backend) { b.Roles = append(b.Roles, RoleDreamEmbed) }, "roles"}, + {"rerank role", func(b *Backend) { b.Roles = append(b.Roles, RoleRerank) }, "roles"}, + {"extra_body effort typo", func(b *Backend) { b.ExtraBody = map[string]any{"effort": "mediumm"} }, "extra_body"}, + {"extra_body effort non-string", func(b *Backend) { b.ExtraBody = map[string]any{"effort": 2} }, "extra_body"}, + {"model_map effort typo", func(b *Backend) { + b.ModelMap = map[string]ModelSpec{"default": {Model: "m", Params: map[string]any{"effort": "ultra"}}} + }, "model_map"}, + {"versioned root", func(b *Backend) { b.Host = "https://api.anthropic.com/v1" }, "base_url"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b := anthropicBackend() + tc.mut(&b) + if _, errs := ValidateBackend(&b); !fieldHit(errs, tc.field) { + t.Errorf("want a %q error, got %v", tc.field, errs) + } + }) + } + + // Valid efforts pass in both carriers. + for _, e := range []string{"low", "medium", "high", "xhigh", "max"} { + b := anthropicBackend() + b.ExtraBody = map[string]any{"effort": e} + b.ModelMap = map[string]ModelSpec{"default": {Model: "m", Params: map[string]any{"effort": e}}} + if _, errs := ValidateBackend(&b); len(errs) != 0 { + t.Errorf("effort %q rejected: %v", e, errs) + } + } + + // The rules are protocol-bound: an openai row keeps its freedoms (keyless, + // no num_ctx, embed roles on a verified external backend are the existing + // contracts and must not regress). + b := validBackend() + b.NumCtx = 0 + if _, errs := ValidateBackend(&b); fieldHit(errs, "num_ctx") || fieldHit(errs, "api_key_ref") { + t.Errorf("anthropic rules leaked onto an openai row: %v", errs) + } +} diff --git a/go/internal/schemacontract/manifest.json b/go/internal/schemacontract/manifest.json index c055bea1..c92d0413 100644 --- a/go/internal/schemacontract/manifest.json +++ b/go/internal/schemacontract/manifest.json @@ -1,8 +1,8 @@ { "manifest_version": 1, "generated_against": { - "migration_max": 154, - "migration_count": 42, + "migration_max": 155, + "migration_count": 43, "pg_major": 18, "image": "pgvector-timescaledb:pg18" }, diff --git a/go/migrations/155_backend_protocol_anthropic.sql b/go/migrations/155_backend_protocol_anthropic.sql new file mode 100644 index 00000000..8318b23e --- /dev/null +++ b/go/migrations/155_backend_protocol_anthropic.sql @@ -0,0 +1,41 @@ +-- ============================================================================= +-- 155_backend_protocol_anthropic.sql — 'anthropic' joins the backend wire +-- protocols +-- Part of ctx by GottZ (https://github.com/GottZ/ctx) +-- ============================================================================= +-- The baseline (113, context_backends) pins the protocol column to the three +-- wires the daemon spoke when the pool was designed: openai, ollama, rerank. +-- The native Anthropic Messages API wire (go/internal/llm/anthropic.go) is a +-- fourth; without this migration the CHECK rejects the row the handler's +-- validation already accepted, and the create fails one layer too late with +-- a bare constraint error. +-- +-- The constraint was declared inline and unnamed in the baseline, so its name +-- is whatever the server assigned (context_backends_protocol_check on every +-- PostgreSQL this project supports). The DO block resolves it from the catalog +-- by table + definition instead of trusting the derived name, so a database +-- whose constraint was ever renamed by hand migrates the same way. +-- +-- Nothing else changes: the column keeps its DEFAULT 'openai', the schema +-- manifest records column types only (no CHECK clauses), and existing rows are +-- untouched. Rolling back the daemon to a build without the anthropic wire +-- leaves such a row in the table; it would then dispatch onto the Ollama path +-- (pre-155 fallthrough) — delete or disable anthropic rows before downgrading. + +DO $$ +DECLARE + con_name text; +BEGIN + SELECT conname INTO con_name + FROM pg_constraint + WHERE conrelid = 'context_backends'::regclass + AND contype = 'c' + AND pg_get_constraintdef(oid) LIKE '%protocol%'; + IF con_name IS NOT NULL THEN + EXECUTE format('ALTER TABLE context_backends DROP CONSTRAINT %I', con_name); + END IF; + ALTER TABLE context_backends + ADD CONSTRAINT context_backends_protocol_check + CHECK (protocol IN ('openai','ollama','rerank','anthropic')); +END +$$; From be2dda79cda8d3cc5ce4c0d7c00ff1d5ed2c8637 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 00:12:47 +0200 Subject: [PATCH 05/16] feat(llm): native Anthropic Messages API wire, non-streaming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chatWithFormat gains a third leg: ProtocolAnthropic dispatches to chatAnthropic (anthropic.go), POST {base_url}/v1/messages with x-api-key and anthropic-version headers (core headers set last, after extra_headers, as setChatHeaders does). The request carries the system prompt as a top-level text block with cache_control ephemeral, the user prompt as the single message, max_tokens from NumPredict (16000 when the caller sets none — the field is required on this wire), and output_config.effort resolved from the role's model_map params (Options.Extra) over the row's extra_body over the explicit default medium. No sampling parameter is ever marshalled: the Claude 5 models answer 400 to temperature/top_p/top_k, and the OpenAI-compatible layer ctx used until now needed a stripping sidecar for exactly that. The think toggle is ignored (thinking cannot be disabled on Opus 5.5; effort is the depth control), logprobs are never requested (the decide primitive falls back through its existing ErrNoLogprobs memo), and ChatJSON is a documented no-op (output_config.format needs a JSON schema ctx does not have; the pipelines' parsers stay the validator, the CTX_DREAM_JSON_MODE=off contract). Other extra_body keys merge top-level, row over params. Response: text blocks joined, thinking blocks skipped; PromptTokens is the TOTAL prompt (input_tokens + cache_read + cache_creation — Anthropic's input_tokens is the uncached remainder) so llmlog's prompt_tokens stays comparable across wires; the two cache counters travel in new ChatResponse fields and reach llmlog metadata through ApplyProviderTelemetry (cache_read_input_tokens / cache_creation_input_tokens, written only when non-zero so other wires' rows stay byte-identical); the message id lands in provider_request_id; cost stays NULL. stop_reason is mapped onto the vocabulary the consumers read (end_turn→stop, max_tokens→length, tool_use→tool_calls, refusal→content_filter, unknown raw, absent ""), because dream's cap detection compares against "length" and the chat harness against "tool_calls" — a raw pass-through would hide both. Non-2xx answers wrap httpx.StatusError as on every wire (429 → RateLimit with Retry-After, 529 overloaded → Gateway failover, 401 → Auth, 400 → BadRequest, 500 → ServerFault) with the provider's error type folded into the message. Tests (httptest fakes, no live key): request shape and headers, effort precedence, extra_body merge, JSON no-op, max_tokens fallback, empty system omitted, response decode incl. cache counters and telemetry, stop-reason table, error classification; anthropic rows in the wire-path and finish-reason tables; the MW6 dispatch invariant gains the seventh wire site (llm/anthropic.go pinned, Σ acquires == Σ wire contacts). docs/api.md: the Anthropic paragraph under Backend pool. Co-Authored-By: Claude Fable 5.1 --- docs/api.md | 4 +- go/internal/chat/dispatch_invariant_test.go | 22 +- go/internal/llm/anthropic.go | 338 +++++++++++++++++ go/internal/llm/anthropic_wire_test.go | 385 ++++++++++++++++++++ go/internal/llm/chain.go | 29 ++ go/internal/llm/client.go | 34 +- go/internal/llm/finishreason_wire_test.go | 8 + go/internal/llm/wire_test.go | 9 + 8 files changed, 812 insertions(+), 17 deletions(-) create mode 100644 go/internal/llm/anthropic.go create mode 100644 go/internal/llm/anthropic_wire_test.go diff --git a/docs/api.md b/docs/api.md index 3bba05b6..23060fae 100644 --- a/docs/api.md +++ b/docs/api.md @@ -441,7 +441,9 @@ message — the verdict lives in the body, like every `backend-test` outcome. **Validation guards** (create AND update, `422` with field errors): credential-carrier headers in `extra_headers` (`Authorization`, `Cookie`, `*-key`, `*-token`, …) and credential-semantic `extra_body` fields are rejected — provider keys go through `api_key_ref` (the *name* of a sealed secret, resolved in-memory only); a `base_url` carrying userinfo (`user:pass@host`) is rejected for the same reason — the URL is echoed unredacted by every list response, log line and the append-only audit (rows written before this guard load fine and are rejected on their next write); `locality` is cross-validated against `base_url` (a publicly routable host must be `external`); embed roles on external backends are blocked without `metadata.embed_equivalence_verified=true` (foreign quantization corrupts the shared vector space). `metadata.score_domain` (rerank dialect slot) must be one of `auto` | `logit` | `probability` — `auto` (the default for every row that does not set it) keeps the wire-container coupling: `results` ⇒ raw logits, `data` ⇒ calibrated [0,1] probability; explicit values override the coupling for dialect-mixing backends. Changing the *effective* score domain of a rerank-capable backend (create with a non-`auto` value included — the delete+create bypass is closed like the trust confirm's) requires `confirm_score_domain_change:true`; the 400 carries the full chain of change (effective immediately on all future rankings; the access-log score trail is only comparable up to the switch; no corpus reprocessing — no curation artifact persists rerank-derived data) and the confirmed write echoes it as a warning. Raising `trust` requires `confirm_trust_elevation:true`. Every mutation reloads the pool snapshot synchronously — `backend-update {"enabled":false}` is an instant brake, no restart; psql edits converge via the 053 NOTIFY trigger. -**OpenRouter (first external backend, G29).** `provider_class: "openrouter"` refines the openai wire: the request **always** carries `provider.zdr:true` + `provider.data_collection:"deny"`, independent of trust level — trust decides WHICH content may flow, the provider class decides whether the provider may store it. `extra_body.provider` entries merge but can only tighten (the force runs after the merge). The single escape is `metadata.allow_data_collection:true`, requiring `confirm_data_collection:true`. Responses feed telemetry: `usage.cost` → llmlog `cost_usd` (local NULL), the top-level `model` (that actually answered) overwrites the row's model, the response `id` lands in `metadata.provider_request_id`. A "no providers" rejection (zdr/deny filter leaves no provider) classifies as configuration-permanent (1h cooldown, no retry storm). `backend-test` additionally reports `credits_remaining`/`usage_usd` (`GET /v1/key`) and `zdr_endpoints` (`GET /v1/endpoints/zdr`). **`base_url` is the API root WITHOUT the version segment** — the wire paths append `/v1/...` themselves; a `base_url` ending in `/v1` double-segments to a 404, and since v4.25.4 create/update reject such a root with a 422 (all openai- and rerank-protocol backends, not just openrouter). +**OpenRouter (first external backend, G29).** `provider_class: "openrouter"` refines the openai wire: the request **always** carries `provider.zdr:true` + `provider.data_collection:"deny"`, independent of trust level — trust decides WHICH content may flow, the provider class decides whether the provider may store it. `extra_body.provider` entries merge but can only tighten (the force runs after the merge). The single escape is `metadata.allow_data_collection:true`, requiring `confirm_data_collection:true`. Responses feed telemetry: `usage.cost` → llmlog `cost_usd` (local NULL), the top-level `model` (that actually answered) overwrites the row's model, the response `id` lands in `metadata.provider_request_id`. A "no providers" rejection (zdr/deny filter leaves no provider) classifies as configuration-permanent (1h cooldown, no retry storm). `backend-test` additionally reports `credits_remaining`/`usage_usd` (`GET /v1/key`) and `zdr_endpoints` (`GET /v1/endpoints/zdr`). **`base_url` is the API root WITHOUT the version segment** — the wire paths append `/v1/...` themselves; a `base_url` ending in `/v1` double-segments to a 404, and since v4.25.4 create/update reject such a root with a 422 (all openai-, anthropic- and rerank-protocol backends, not just openrouter). + +**Anthropic (native Messages API, Migration 155).** `protocol: "anthropic"` speaks `POST {base_url}/v1/messages` directly (`base_url: "https://api.anthropic.com"`, the key from `api_key_ref` travels as `x-api-key`, `anthropic-version: 2023-06-01`) instead of Anthropic's OpenAI-compatible layer — which rejects every sampling parameter on the Claude 5 models, drops `response_format` silently and has neither prompt caching nor effort control. The request carries **no sampling parameters** (`temperature`/`top_p`/`top_k` are 400s on these models; `num_ctx` stays ctx's own prompt budget), the system prompt as a top-level block with `cache_control: ephemeral` (cache hits need a ≥512-token prefix; verify in `metadata.cache_read_input_tokens` of the `context_llm_log` row), `max_tokens` from the pipeline's `num_predict` (**thinking tokens count against it** — the Claude 5 models think adaptively and cannot be told not to, so a model_map `think:false` is ignored on this wire), and `output_config.effort`: the role's `model_map` `params.effort` beats the row's `extra_body.effort`, which beats the explicit default `medium` (the model-side default of Claude Opus 5.5); both carriers accept only `low|medium|high|xhigh|max` (422 otherwise). Every other `extra_body` key merges top-level (the hatch for `thinking.display`, `stop_sequences`, …; the row wins over model_map params on collision, as on the openai wire). **JSON mode is a no-op** on this wire: `output_config.format` (structured outputs) needs a JSON schema with `additionalProperties:false` on every object, which ctx's schemaless `json_object` mode does not have, so `ChatJSON` sends the same body as `Chat` and the pipelines' local parsers stay the validator — run Dream with `CTX_DREAM_JSON_MODE=off` (see [operations](operations.md#environment-variables)). Responses: text blocks are concatenated (thinking blocks skipped), `prompt_tokens` is the **total** prompt (`input_tokens` + `cache_read_input_tokens` + `cache_creation_input_tokens` — Anthropic's `input_tokens` is the uncached remainder only), the two cache counters land in `metadata`, the message id in `metadata.provider_request_id`, `cost_usd` stays NULL (ctx does not price Anthropic; read the console), and `stop_reason` is mapped onto the vocabulary the consumers read (`end_turn`→`stop`, `max_tokens`→`length`, `tool_use`→`tool_calls`, `refusal`→`content_filter`; unknown values pass through raw). Errors classify like every other wire: 429 → rate-limit cooldown (`Retry-After` honoured), 529 `overloaded_error` → gateway (failover), 401 → auth, 400 → request-specific; the provider's error type prefixes the logged message. Row shape (422 on create and update): `api_key_ref` required, `locality: external`, `num_ctx > 0` (no window discovery on this wire — synthesis would fail closed), `provider_class: generic`, no `embed`/`dream-embed`/`rerank` roles (the Messages API has no such endpoints). `backend-test` probes `GET {base_url}/v1/models` with the key, then the 1-token chat. The web-chat stream path (`ChatStreamBackend`) maps the Messages API SSE events (`content_block_delta` text/`input_json_delta`, `message_delta` stop reason + usage, `error`) onto the same event contract as the openai wire, tools included (`tool_use`/`tool_result` blocks, consecutive tool results folded into one user turn). Not available on this wire by construction: logprobs (the dream decide primitive falls back to its generating prompt and memoises the backend as decide-incapable), forced `tool_choice` (a 400 on Claude Opus 5.5; `auto` is what both wires send). ## Web chat sessions diff --git a/go/internal/chat/dispatch_invariant_test.go b/go/internal/chat/dispatch_invariant_test.go index fe7f3d19..796b166c 100644 --- a/go/internal/chat/dispatch_invariant_test.go +++ b/go/internal/chat/dispatch_invariant_test.go @@ -125,6 +125,8 @@ func newWireServer(t *testing.T, wireHits *int64) *httptest.Server { fmt.Fprintf(w, `{"embeddings":[[%s]],"prompt_eval_count":7}`, vec) case "/v1/embeddings": // embed.go:185 (openai) fmt.Fprintf(w, `{"data":[{"embedding":[%s]}]}`, vec) + case "/v1/messages": // anthropic.go (native Messages API, non-stream) + fmt.Fprint(w, `{"id":"msg_1","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":10,"output_tokens":5}}`) case "/v1/rerank": // rerank.go:96 var req struct { Documents []string `json:"documents"` @@ -212,9 +214,16 @@ func TestDispatchAcquireEqualsWireAllSites(t *testing.T) { t.Fatalf("rerank: wired=%v err=%v", tel.Wired, err) } - // Six sites, each one physical wire contact under one lease. - if ca.count() != 6 || atomic.LoadInt64(&wireHits) != 6 { - t.Fatalf("after the six sites: acquires=%d wire=%d, want 6/6", ca.count(), atomic.LoadInt64(&wireHits)) + // --- Site 7: anthropic.go (native Messages API, non-stream) — reached + // through the same chain dispatcher as sites 1 and 2 (chatWithFormat). --- + if _, _, _, err := llm.ChatChain(ctx, []backends.Backend{invariantBackend(srv.URL, backends.ProtocolAnthropic)}, + "chat", "s", "u", llm.Options{}, "", time.Second, nil, llmAdm); err != nil { + t.Fatalf("anthropic non-stream: %v", err) + } + + // Seven sites, each one physical wire contact under one lease. + if ca.count() != 7 || atomic.LoadInt64(&wireHits) != 7 { + t.Fatalf("after the seven sites: acquires=%d wire=%d, want 7/7", ca.count(), atomic.LoadInt64(&wireHits)) } // --- Failover walk (design W5: "inkl. Failover-Pfade"): a down primary @@ -231,8 +240,8 @@ func TestDispatchAcquireEqualsWireAllSites(t *testing.T) { "chat", "s", "u", llm.Options{}, "", time.Second, nil, llmAdm); err != nil || served == nil || len(attempts) != 2 { t.Fatalf("failover walk: served=%v attempts=%d err=%v", served, len(attempts), err) } - if ca.count() != 8 || atomic.LoadInt64(&wireHits) != 8 { - t.Fatalf("after failover: acquires=%d wire=%d, want 8/8", ca.count(), atomic.LoadInt64(&wireHits)) + if ca.count() != 9 || atomic.LoadInt64(&wireHits) != 9 { + t.Fatalf("after failover: acquires=%d wire=%d, want 9/9", ca.count(), atomic.LoadInt64(&wireHits)) } // --- Lease-free without wire contact (the cache-hit clause's structural @@ -263,12 +272,13 @@ func TestDispatchAcquireEqualsWireAllSites(t *testing.T) { var wirePin = map[string]int{ "llm/client.go": 2, // /api/chat + /v1/chat/completions (non-stream) "llm/stream.go": 1, // /v1/chat/completions (stream) + "llm/anthropic.go": 1, // /v1/messages (anthropic non-stream) "embed/embed.go": 2, // /api/embed + /v1/embeddings "rerank/rerank.go": 1, // /v1/rerank } // TestWireSitePositiveListPinned pins the grep positive list: the ONLY -// http.NewRequestWithContext sites in internal/{llm,embed,rerank} are the six +// http.NewRequestWithContext sites in internal/{llm,embed,rerank} are the ones // listed above. A new un-listed wire file, or a changed count in a listed file, // turns this gate red — the static half of the I-D1 endpoint proof (§1.2: // admittiertheit itself is proven by the count gate, not by grep). diff --git a/go/internal/llm/anthropic.go b/go/internal/llm/anthropic.go new file mode 100644 index 00000000..2e9eb8af --- /dev/null +++ b/go/internal/llm/anthropic.go @@ -0,0 +1,338 @@ +// Native Anthropic Messages API wire (backends.ProtocolAnthropic): POST +// {base_url}/v1/messages with x-api-key + anthropic-version, the system +// prompt as a top-level cached block, and NO sampling parameters. +// +// Why a third wire instead of the openai path against Anthropic's +// OpenAI-compatible layer: that layer rejects every sampling parameter on +// the Claude 5 models (400 "temperature is deprecated for this model" — and +// chatOpenAI always sends temperature), drops response_format silently, has +// no prompt caching and no effort control, and Anthropic documents it as a +// comparison surface, not a production one. The ku-heimdall install ran a +// stripping sidecar in front of it for exactly those reasons; this file is +// what makes that sidecar unnecessary. +// +// Model-side facts this wire is built on (claude-api reference, 2026-10, +// Claude Opus 5.5 / Sonnet 5.5 / Fable 5.1 family): +// - temperature / top_p / top_k → 400: none of Options' sampling fields +// are ever marshalled; num_ctx stays a ctx-internal prompt budget. +// - thinking cannot be disabled (Opus 5.5: {type:"disabled"} is a 400 at +// every effort); omitting the field runs adaptive thinking. The +// model_map `think` toggle is therefore ignored here, and depth is +// steered with output_config.effort instead (resolveAnthropicEffort). +// - forced tool_choice (any/tool) → 400: never sent (auto is the default, +// matching buildStreamBody's doctrine on the openai wire). +// - output_config.format needs a JSON schema with additionalProperties: +// false on every object; ctx's "json" format carries no schema and the +// dream stages answer with arrays, so ChatJSON is a documented no-op on +// this wire — the pipelines' local parsers stay the validator, exactly +// the CTX_DREAM_JSON_MODE=off contract (docs/operations.md). +// - max_tokens is REQUIRED and thinking tokens count against it. +// - usage.input_tokens is the UNCACHED remainder; the total prompt is +// input + cache_read + cache_creation (anthropicUsageTotals), which is +// what PromptTokens reports so llmlog's prompt_tokens stays comparable +// with the other wires. The two cache counters travel separately for +// the llmlog metadata (ApplyProviderTelemetry). +// - 529 overloaded_error is a plain non-2xx: httpx.StatusError carries it +// into backends.Classify as ClassGateway (failover), 429 as +// ClassRateLimit with the Retry-After cooldown, 401 as ClassAuth. +package llm + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/httpx" +) + +const ( + // anthropicVersion is the Messages API version header. 2023-06-01 is the + // only GA value; features ship behind anthropic-beta, not new versions. + anthropicVersion = "2023-06-01" + // anthropicDefaultMaxTokens applies when the caller sets no NumPredict: + // the field is required on this wire, and every production Options + // preset sets one (synthesis 500, translate 100, probe 1), so this is + // the safety net for a bare Options{}, sized so a long answer is not + // cut mid-thought (claude-api reference default for non-streaming). + anthropicDefaultMaxTokens = 16000 + // anthropicEffortKey is the model_map params / extra_body key that is + // LIFTED into output_config.effort instead of being merged top-level. + anthropicEffortKey = "effort" +) + +// --- request shape ---. + +type anthropicCacheControl struct { + Type string `json:"type"` // "ephemeral" +} + +// anthropicTextBlock is one text content block; CacheControl marks the +// prefix breakpoint (the system prompt here). +type anthropicTextBlock struct { + Type string `json:"type"` + Text string `json:"text"` + CacheControl *anthropicCacheControl `json:"cache_control,omitempty"` +} + +// anthropicMessage is one turn. Content is either a plain string (the +// single-user-prompt pipelines) or a []any of content blocks (the chat +// harness: text, tool_use, tool_result). +type anthropicMessage struct { + Role string `json:"role"` + Content any `json:"content"` +} + +type anthropicOutputConfig struct { + Effort string `json:"effort"` +} + +// anthropicTool is the Messages API tool definition (input_schema, not the +// OpenAI function envelope). +type anthropicTool struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + InputSchema json.RawMessage `json:"input_schema"` +} + +type anthropicRequest struct { + Model string `json:"model"` + MaxTokens int `json:"max_tokens"` + System []anthropicTextBlock `json:"system,omitempty"` + Messages []anthropicMessage `json:"messages"` + Stream bool `json:"stream,omitempty"` + OutputConfig anthropicOutputConfig `json:"output_config"` + Tools []anthropicTool `json:"tools,omitempty"` +} + +// --- response shape ---. + +type anthropicUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheReadInputTokens int `json:"cache_read_input_tokens"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens"` +} + +// anthropicContentBlock is one response block. text carries Text; +// tool_use carries ID/Name/Input; thinking blocks are skipped (their text +// is empty under the default display anyway). +type anthropicContentBlock struct { + Type string `json:"type"` + Text string `json:"text"` + ID string `json:"id"` + Name string `json:"name"` + Input json.RawMessage `json:"input"` +} + +type anthropicResponse struct { + ID string `json:"id"` + Model string `json:"model"` + StopReason string `json:"stop_reason"` + Content []anthropicContentBlock `json:"content"` + Usage anthropicUsage `json:"usage"` +} + +// anthropicError is the body of every non-2xx answer and of a mid-stream +// error event: {"type":"error","error":{"type":"...","message":"..."}}. +type anthropicError struct { + Error struct { + Type string `json:"type"` + Message string `json:"message"` + } `json:"error"` +} + +// --- helpers shared by the non-streaming and the streaming path ---. + +// resolveAnthropicEffort picks output_config.effort: the role's model_map +// params (arriving in Options.Extra through applyModelParams' default +// branch) beat the row's extra_body, which beats the explicit default. +// Validation (backends.validateAnthropic) has already bounded both carriers +// to the accepted set; a non-string value here falls through to the next +// carrier rather than reaching the wire. +func resolveAnthropicEffort(opts Options, b *backends.Backend) string { + if v, ok := opts.Extra[anthropicEffortKey].(string); ok && v != "" { + return v + } + if v, ok := b.ExtraBody[anthropicEffortKey].(string); ok && v != "" { + return v + } + return backends.AnthropicDefaultEffort +} + +func anthropicMaxTokens(opts Options) int { + if opts.NumPredict > 0 { + return opts.NumPredict + } + return anthropicDefaultMaxTokens +} + +// anthropicSystem renders the system prompt as the cached top-level block. +// An empty prompt yields no system field (the API rejects an empty text +// block). +func anthropicSystem(systemPrompt string) []anthropicTextBlock { + if systemPrompt == "" { + return nil + } + return []anthropicTextBlock{{ + Type: "text", Text: systemPrompt, + CacheControl: &anthropicCacheControl{Type: "ephemeral"}, + }} +} + +// buildAnthropicBody marshals the request and merges the two extra carriers +// — Options.Extra (model_map params) first, b.ExtraBody last, so a backend +// row's extra_body keeps the final word exactly as on the openai wire +// (applyOpenAIBodyExtras). The effort key is consumed by +// resolveAnthropicEffort and never merged top-level (it is not a Messages +// API field). Everything else merges verbatim: that is the operator's +// escape hatch for fields this wire does not model (thinking.display, +// stop_sequences, metadata.user_id, …). +func buildAnthropicBody(req anthropicRequest, opts Options, b *backends.Backend) ([]byte, error) { + body, err := json.Marshal(req) + if err != nil { + return nil, err + } + extra := make(map[string]any, len(opts.Extra)+len(b.ExtraBody)) + for k, v := range opts.Extra { + if k != anthropicEffortKey { + extra[k] = v + } + } + for k, v := range b.ExtraBody { + if k != anthropicEffortKey { + extra[k] = v + } + } + return mergeJSONFields(body, extra) +} + +// setAnthropicHeaders mirrors setChatHeaders' doctrine: ExtraHeaders first, +// the core headers LAST so a row edited past the credential-carrier +// denylist can never override the key derived from api_key_ref or the +// version pin. +func setAnthropicHeaders(req *http.Request, b *backends.Backend) { + for k, v := range b.ExtraHeaders { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("anthropic-version", anthropicVersion) + if b.APIKey != "" { + req.Header.Set("x-api-key", b.APIKey) + } +} + +// mapAnthropicStopReason translates the Messages API vocabulary into the +// one every FinishReason consumer already compares against (dream's cap +// detection reads "length", the chat harness reads "tool_calls"): a raw +// pass-through — the doctrine for the ollama/openai pair, whose +// vocabularies overlap — would make an Anthropic cap hit invisible to +// evaluate.go and a tool call look like a final answer to engine.go. Unknown +// values pass through raw so a new provider state is observable in llmlog; +// an absent reason stays "" (unknown), never a fabricated "stop". +func mapAnthropicStopReason(s string) string { + switch s { + case "": + return "" + case "end_turn", "stop_sequence", "pause_turn": + return "stop" + case "max_tokens", "model_context_window_exceeded": + return "length" + case "tool_use": + return "tool_calls" + case "refusal": + return "content_filter" + default: + return s + } +} + +// anthropicUsageTotals folds the usage object into ctx's counters: the +// prompt total (uncached remainder + cache read + cache write) and the +// output count, plus the two cache counters for the llmlog metadata. +func anthropicUsageTotals(u anthropicUsage) (prompt, output, cacheRead, cacheCreate int) { + return u.InputTokens + u.CacheReadInputTokens + u.CacheCreationInputTokens, + u.OutputTokens, u.CacheReadInputTokens, u.CacheCreationInputTokens +} + +// anthropicStatusError wraps a non-2xx answer the way the other wires do +// (httpx.StatusError for the failover classifier, rendered string unchanged) +// and prefixes the provider's error type when the body parses, so a log +// line reads "overloaded_error: Overloaded" instead of raw JSON. +func anthropicStatusError(resp *http.Response) error { + errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + var ae anthropicError + if json.Unmarshal(errBody, &ae) == nil && ae.Error.Type != "" { + errBody = []byte(ae.Error.Type + ": " + ae.Error.Message) + } + return fmt.Errorf("llm: %w", httpx.NewStatusError(resp, errBody)) +} + +// --- non-streaming path ---. + +// chatAnthropic is the ProtocolAnthropic leg of chatWithFormat. format is +// accepted for signature parity and ignored (see the package comment on +// structured outputs); think is ignored (thinking cannot be disabled). +func chatAnthropic(ctx context.Context, b backends.Backend, systemPrompt, userPrompt string, opts Options, _ string, timeout time.Duration) (*ChatResponse, error) { + req := anthropicRequest{ + Model: b.Model, + MaxTokens: anthropicMaxTokens(opts), + System: anthropicSystem(systemPrompt), + Messages: []anthropicMessage{{Role: "user", Content: userPrompt}}, + OutputConfig: anthropicOutputConfig{Effort: resolveAnthropicEffort(opts, &b)}, + } + body, err := buildAnthropicBody(req, opts, &b) + if err != nil { + return nil, fmt.Errorf("llm: marshal request: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, b.Host+"/v1/messages", bytes.NewReader(body)) //nolint:gosec // G704: host from the F3 backend pool (context_backends) — admin-gated + locality-validated, no free user input. + if err != nil { + return nil, fmt.Errorf("llm: create request: %w", err) + } + setAnthropicHeaders(httpReq, &b) + + resp, err := httpx.DoRetryOnce(httpClient, httpReq, body) + if err != nil { + return nil, fmt.Errorf("llm: request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, anthropicStatusError(resp) + } + + var result anthropicResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("llm: decode response: %w", err) + } + + var text strings.Builder + for _, blk := range result.Content { + if blk.Type == "text" { + text.WriteString(blk.Text) + } + } + prompt, output, cacheRead, cacheCreate := anthropicUsageTotals(result.Usage) + return &ChatResponse{ + Message: Message{Role: "assistant", Content: text.String()}, + EvalCount: output, + PromptTokens: prompt, + CacheReadTokens: cacheRead, + CacheCreationTokens: cacheCreate, + FinishReason: mapAnthropicStopReason(result.StopReason), + // The message id is the provider's audit handle (support requests + // quote it); the model column stays row-faithful — Anthropic echoes + // the requested id, there is no models-fallback to record — and no + // cost is reported on this wire (billing is read in the console). + ProviderRequestID: result.ID, + }, nil +} diff --git a/go/internal/llm/anthropic_wire_test.go b/go/internal/llm/anthropic_wire_test.go new file mode 100644 index 00000000..942dd3f1 --- /dev/null +++ b/go/internal/llm/anthropic_wire_test.go @@ -0,0 +1,385 @@ +// Native Anthropic Messages API wire (anthropic.go), non-streaming path: +// request shape, headers, effort resolution, extra_body merge, response +// decode (text/thinking blocks, usage incl. cache counters, stop-reason +// mapping, message id) and the error → failover-class mapping. Every probe +// runs against an httptest fake of api.anthropic.com — no live key in CI. +// What this file CANNOT prove (live only): an actual cache hit, the effort +// level's effect on output_tokens, 529 behaviour under load. +package llm + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/httpx" + "github.com/GottZ/ctx/internal/llmlog" +) + +// anthropicRecorder captures path, decoded body and headers of one +// /v1/messages call and answers with a fixed JSON (or a fixed status). +type anthropicRecorder struct { + mu sync.Mutex + path string + body map[string]any + header http.Header + srv *httptest.Server +} + +func newAnthropicRecorder(t *testing.T, status int, respJSON string, respHeader map[string]string) *anthropicRecorder { + t.Helper() + rec := &anthropicRecorder{} + rec.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var m map[string]any + _ = json.NewDecoder(r.Body).Decode(&m) + rec.mu.Lock() + rec.path = r.URL.Path + rec.body = m + rec.header = r.Header.Clone() + rec.mu.Unlock() + for k, v := range respHeader { + w.Header().Set(k, v) + } + w.WriteHeader(status) + _, _ = w.Write([]byte(respJSON)) + })) + t.Cleanup(rec.srv.Close) + return rec +} + +func (rec *anthropicRecorder) recorded() (string, map[string]any, http.Header) { + rec.mu.Lock() + defer rec.mu.Unlock() + return rec.path, rec.body, rec.header +} + +// anthropicRespOK is a complete Messages API answer: a thinking block (empty +// text, the default display), two text blocks, cache counters on usage. +const anthropicRespOK = `{"id":"msg_01abc","type":"message","role":"assistant","model":"claude-opus-5-5", + "content":[{"type":"thinking","thinking":"","signature":"sig"}, + {"type":"text","text":"Hel"},{"type":"text","text":"lo"}], + "stop_reason":"end_turn","stop_sequence":null, + "usage":{"input_tokens":10,"output_tokens":7, + "cache_read_input_tokens":400,"cache_creation_input_tokens":100}}` + +func anthropicTestBackend(host string) backends.Backend { + return backends.Backend{ + Host: host, Protocol: backends.ProtocolAnthropic, + ProviderClass: backends.ProviderGeneric, Model: "claude-opus-5-5", + Trust: backends.TrustNoCredentials, Locality: backends.LocalityExternal, + APIKey: "sk-ant-test", APIKeyRef: "anthropic-chat", NumCtx: 100000, + } +} + +// sampling keys the Claude 5 models answer 400 to, plus every openai-wire +// field that has no meaning on this wire. None may ever reach the body. +var anthropicForbiddenKeys = []string{ + "temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", + "repeat_penalty", "logprobs", "top_logprobs", "response_format", "reasoning", "seed", + "thinking", "tool_choice", "stream_options", "options", "num_ctx", "effort", +} + +func TestAnthropicRequestWire(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + b.Think = "false" // ignored on this wire — thinking cannot be disabled + b.ExtraHeaders = map[string]string{"X-Title": "ctx", "anthropic-version": "1999-01-01"} + opts := Options{Temperature: 0.1, TopP: 0.9, TopK: 40, RepeatPenalty: 1.1, NumPredict: 5, NumCtx: 100000, TopLogprobs: 20} + if _, err := Chat(context.Background(), b, "sys prompt", "user prompt", opts, 5*time.Second); err != nil { + t.Fatalf("Chat: %v", err) + } + path, body, hdr := rec.recorded() + + if path != "/v1/messages" { + t.Errorf("path = %q, want /v1/messages", path) + } + if got := hdr.Get("x-api-key"); got != "sk-ant-test" { + t.Errorf("x-api-key = %q, want the resolved api_key_ref", got) + } + if got := hdr.Get("anthropic-version"); got != anthropicVersion { + t.Errorf("anthropic-version = %q, want %q (core header must beat extra_headers)", got, anthropicVersion) + } + if got := hdr.Get("Authorization"); got != "" { + t.Errorf("Authorization = %q, want none (the Messages API authenticates via x-api-key)", got) + } + if got := hdr.Get("X-Title"); got != "ctx" { + t.Errorf("X-Title = %q, want extra_headers to reach the wire", got) + } + if got := hdr.Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type = %q", got) + } + + if body["model"] != "claude-opus-5-5" { + t.Errorf("model = %v", body["model"]) + } + if body["max_tokens"] != float64(5) { + t.Errorf("max_tokens = %v, want NumPredict 5", body["max_tokens"]) + } + if _, has := body["stream"]; has { + t.Errorf("stream key present on the non-streaming path: %v", body["stream"]) + } + sys, _ := body["system"].([]any) + if len(sys) != 1 { + t.Fatalf("system = %v, want one cached text block", body["system"]) + } + blk, _ := sys[0].(map[string]any) + if blk["type"] != "text" || blk["text"] != "sys prompt" { + t.Errorf("system block = %v", blk) + } + if cc, _ := blk["cache_control"].(map[string]any); cc["type"] != "ephemeral" { + t.Errorf("system cache_control = %v, want {type: ephemeral}", blk["cache_control"]) + } + msgs, _ := body["messages"].([]any) + if len(msgs) != 1 { + t.Fatalf("messages = %v, want exactly the user turn", body["messages"]) + } + if m, _ := msgs[0].(map[string]any); m["role"] != "user" || m["content"] != "user prompt" { + t.Errorf("messages[0] = %v", msgs[0]) + } + if oc, _ := body["output_config"].(map[string]any); oc["effort"] != backends.AnthropicDefaultEffort { + t.Errorf("output_config = %v, want effort %q by default", body["output_config"], backends.AnthropicDefaultEffort) + } + for _, k := range anthropicForbiddenKeys { + if v, has := body[k]; has { + t.Errorf("body carries %q = %v — the Claude 5 models reject sampling fields and nothing else here is a Messages API field", k, v) + } + } +} + +// TestAnthropicEffortResolution: model_map params (Options.Extra) beat the +// row's extra_body, which beats the default; the key itself never lands +// top-level. +func TestAnthropicEffortResolution(t *testing.T) { + effortOf := func(t *testing.T, b backends.Backend, opts Options) (string, map[string]any) { + t.Helper() + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b.Host = rec.srv.URL + if _, err := Chat(context.Background(), b, "s", "u", opts, 5*time.Second); err != nil { + t.Fatalf("Chat: %v", err) + } + _, body, _ := rec.recorded() + oc, _ := body["output_config"].(map[string]any) + e, _ := oc["effort"].(string) + return e, body + } + + b := anthropicTestBackend("") + b.ExtraBody = map[string]any{"effort": "high"} + if e, body := effortOf(t, b, Options{NumPredict: 5}); e != "high" { + t.Errorf("extra_body effort: got %q, want high", e) + } else if _, leaked := body["effort"]; leaked { + t.Error("effort merged top-level — it must be lifted into output_config only") + } + + if e, _ := effortOf(t, b, Options{NumPredict: 5, Extra: map[string]any{"effort": "low"}}); e != "low" { + t.Errorf("model_map effort must beat extra_body: got %q, want low", e) + } + + if e, _ := effortOf(t, anthropicTestBackend(""), Options{NumPredict: 5}); e != backends.AnthropicDefaultEffort { + t.Errorf("default effort: got %q, want %q", e, backends.AnthropicDefaultEffort) + } +} + +// TestAnthropicExtraBodyMerge: everything but effort merges top-level with +// the row winning over model_map params (openai-wire parity) — the +// operator's hatch for fields this wire does not model. +func TestAnthropicExtraBodyMerge(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + b.ExtraBody = map[string]any{ + "thinking": map[string]any{"type": "adaptive", "display": "summarized"}, + "metadata": map[string]any{"user_id": "row"}, + } + opts := Options{NumPredict: 5, Extra: map[string]any{"metadata": map[string]any{"user_id": "params"}, "stop_sequences": []any{"END"}}} + if _, err := Chat(context.Background(), b, "s", "u", opts, 5*time.Second); err != nil { + t.Fatalf("Chat: %v", err) + } + _, body, _ := rec.recorded() + if th, _ := body["thinking"].(map[string]any); th["display"] != "summarized" { + t.Errorf("extra_body.thinking did not reach the wire: %v", body["thinking"]) + } + if md, _ := body["metadata"].(map[string]any); md["user_id"] != "row" { + t.Errorf("extra_body must win over model_map params on collision: %v", body["metadata"]) + } + if ss, _ := body["stop_sequences"].([]any); len(ss) != 1 || ss[0] != "END" { + t.Errorf("model_map param stop_sequences did not reach the wire: %v", body["stop_sequences"]) + } + if body["max_tokens"] != float64(5) { + t.Errorf("max_tokens = %v, the merge must not disturb the modelled fields", body["max_tokens"]) + } +} + +// TestAnthropicJSONModeIsNoOp pins the documented contract: ChatJSON sends +// the same body as Chat — no response_format, no output_config.format (that +// needs a schema ctx does not have), and the request still succeeds. +func TestAnthropicJSONModeIsNoOp(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + resp, err := ChatJSON(context.Background(), b, "s", "u", Options{NumPredict: 5}, 5*time.Second) + if err != nil { + t.Fatalf("ChatJSON: %v", err) + } + if resp.Message.Content != "Hello" { + t.Errorf("content = %q", resp.Message.Content) + } + _, body, _ := rec.recorded() + if _, has := body["response_format"]; has { + t.Error("response_format sent on the anthropic wire") + } + if oc, _ := body["output_config"].(map[string]any); oc["format"] != nil { + t.Errorf("output_config.format = %v, want none (no schema available)", oc["format"]) + } +} + +// TestAnthropicMaxTokensRequired: max_tokens cannot be omitted on this wire, +// so a bare Options{} gets the documented fallback instead of a 400. +func TestAnthropicMaxTokensRequired(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + if _, err := Chat(context.Background(), b, "s", "u", Options{}, 5*time.Second); err != nil { + t.Fatalf("Chat: %v", err) + } + _, body, _ := rec.recorded() + if body["max_tokens"] != float64(anthropicDefaultMaxTokens) { + t.Errorf("max_tokens = %v, want the fallback %d", body["max_tokens"], anthropicDefaultMaxTokens) + } +} + +// TestAnthropicEmptySystemOmitted: the API rejects an empty text block, so +// an empty system prompt must yield no system field at all. +func TestAnthropicEmptySystemOmitted(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + if _, err := Chat(context.Background(), b, "", "u", Options{NumPredict: 5}, 5*time.Second); err != nil { + t.Fatalf("Chat: %v", err) + } + _, body, _ := rec.recorded() + if _, has := body["system"]; has { + t.Errorf("system = %v, want the field omitted for an empty prompt", body["system"]) + } +} + +// TestAnthropicResponseDecode: text blocks concatenate, thinking blocks are +// skipped, PromptTokens is the TOTAL prompt (uncached + cache read + cache +// write), the cache counters travel separately, the message id is the +// provider request id, and the openrouter-only fields stay zero. +func TestAnthropicResponseDecode(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + b := anthropicTestBackend(rec.srv.URL) + resp, err := Chat(context.Background(), b, "s", "u", Options{NumPredict: 5}, 5*time.Second) + if err != nil { + t.Fatalf("Chat: %v", err) + } + if resp.Message.Role != "assistant" || resp.Message.Content != "Hello" { + t.Errorf("message = %+v, want assistant/Hello (text blocks joined, thinking skipped)", resp.Message) + } + if resp.EvalCount != 7 { + t.Errorf("EvalCount = %d, want output_tokens 7", resp.EvalCount) + } + if resp.PromptTokens != 510 { + t.Errorf("PromptTokens = %d, want 10+400+100 (input_tokens is the uncached remainder only)", resp.PromptTokens) + } + if resp.CacheReadTokens != 400 || resp.CacheCreationTokens != 100 { + t.Errorf("cache counters = %d/%d, want 400/100", resp.CacheReadTokens, resp.CacheCreationTokens) + } + if resp.FinishReason != "stop" { + t.Errorf("FinishReason = %q, want end_turn mapped to stop", resp.FinishReason) + } + if resp.ProviderRequestID != "msg_01abc" { + t.Errorf("ProviderRequestID = %q, want the message id", resp.ProviderRequestID) + } + if resp.ServedModel != "" || resp.CostUSD != nil || resp.TopLogprobs != nil { + t.Errorf("openrouter-only / logprob fields must stay zero: served=%q cost=%v lp=%v", resp.ServedModel, resp.CostUSD, resp.TopLogprobs) + } + + // The cache counters reach llmlog as metadata, nothing else on the row + // changes (the model column stays the pool's pick). + entry := llmlog.Entry{Model: "pool-model"} + ApplyProviderTelemetry(&entry, resp) + if entry.Model != "pool-model" || entry.CostUSD != nil { + t.Errorf("entry model/cost disturbed: %q %v", entry.Model, entry.CostUSD) + } + if entry.Metadata["cache_read_input_tokens"] != 400 || entry.Metadata["cache_creation_input_tokens"] != 100 { + t.Errorf("metadata = %v, want both cache counters", entry.Metadata) + } + if entry.Metadata["provider_request_id"] != "msg_01abc" { + t.Errorf("metadata.provider_request_id = %v", entry.Metadata["provider_request_id"]) + } + + // A wire without cache counters leaves the metadata untouched (other + // wires' rows stay byte-identical). + plain := llmlog.Entry{} + ApplyProviderTelemetry(&plain, &ChatResponse{}) + if plain.Metadata != nil { + t.Errorf("metadata allocated without cache counters: %v", plain.Metadata) + } +} + +// TestAnthropicStopReasonMapping: the Messages API vocabulary folds into the +// one FinishReason consumers compare against (dream reads "length", the chat +// harness reads "tool_calls"); unknown values pass through raw, absent stays +// "". +func TestAnthropicStopReasonMapping(t *testing.T) { + cases := map[string]string{ + "end_turn": "stop", "stop_sequence": "stop", "pause_turn": "stop", + "max_tokens": "length", "model_context_window_exceeded": "length", + "tool_use": "tool_calls", "refusal": "content_filter", + "some_future_reason": "some_future_reason", "": "", + } + for in, want := range cases { + if got := mapAnthropicStopReason(in); got != want { + t.Errorf("map(%q) = %q, want %q", in, got, want) + } + } +} + +// TestAnthropicErrorClassification: a non-2xx answer carries its status into +// the failover classifier exactly like the other wires — 429 keeps the +// Retry-After cooldown, 529 (overloaded) fails over as a gateway fault, 401 +// is auth, 400 is request-specific, 500 stops the chain. The provider's +// error type is folded into the rendered message. +func TestAnthropicErrorClassification(t *testing.T) { + cases := []struct { + status int + errType string + hdr map[string]string + want backends.ErrClass + retry time.Duration + }{ + {429, "rate_limit_error", map[string]string{"Retry-After": "7"}, backends.ClassRateLimit, 7 * time.Second}, + {529, "overloaded_error", nil, backends.ClassGateway, 0}, + {401, "authentication_error", nil, backends.ClassAuth, 0}, + {400, "invalid_request_error", nil, backends.ClassBadRequest, 0}, + {500, "api_error", nil, backends.ClassServerFault, 0}, + } + for _, tc := range cases { + t.Run(tc.errType, func(t *testing.T) { + body := `{"type":"error","error":{"type":"` + tc.errType + `","message":"nope"}}` + rec := newAnthropicRecorder(t, tc.status, body, tc.hdr) + b := anthropicTestBackend(rec.srv.URL) + _, err := Chat(context.Background(), b, "s", "u", Options{NumPredict: 5}, 5*time.Second) + if err == nil { + t.Fatal("want an error") + } + var se *httpx.StatusError + if !errors.As(err, &se) { + t.Fatalf("error is not a *httpx.StatusError: %v", err) + } + if se.Code != tc.status || se.RetryAfter != tc.retry { + t.Errorf("StatusError = code %d retry %s, want %d/%s", se.Code, se.RetryAfter, tc.status, tc.retry) + } + if se.Body != tc.errType+": nope" { + t.Errorf("body = %q, want the provider error type folded in", se.Body) + } + if got := backends.Classify(err, backends.ProviderGeneric); got != tc.want { + t.Errorf("Classify = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/go/internal/llm/chain.go b/go/internal/llm/chain.go index 5879aae7..4825440c 100644 --- a/go/internal/llm/chain.go +++ b/go/internal/llm/chain.go @@ -712,4 +712,33 @@ func ApplyProviderTelemetry(entry *llmlog.Entry, resp *ChatResponse) { } entry.Metadata["provider_request_id"] = resp.ProviderRequestID } + // Prompt-cache counters (anthropic wire): recorded only when non-zero so + // the other wires' rows stay byte-identical. prompt_tokens already holds + // the total; these make the hit rate readable per row. + applyCacheTelemetry(entry, resp.CacheReadTokens, resp.CacheCreationTokens) +} + +// applyCacheTelemetry writes the prompt-cache counters into the llmlog +// metadata. Shared by the non-streaming funnel (ApplyProviderTelemetry) and +// the web-chat stream row (chat/engine.go recordLLM). +func applyCacheTelemetry(entry *llmlog.Entry, cacheRead, cacheCreate int) { + if cacheRead == 0 && cacheCreate == 0 { + return + } + if entry.Metadata == nil { + entry.Metadata = map[string]any{} + } + if cacheRead > 0 { + entry.Metadata["cache_read_input_tokens"] = cacheRead + } + if cacheCreate > 0 { + entry.Metadata["cache_creation_input_tokens"] = cacheCreate + } +} + +// ApplyCacheTelemetry is the exported form for the stream path (chat/engine), +// which assembles its llmlog row from a StreamResult rather than a +// ChatResponse. +func ApplyCacheTelemetry(entry *llmlog.Entry, cacheRead, cacheCreate int) { + applyCacheTelemetry(entry, cacheRead, cacheCreate) } diff --git a/go/internal/llm/client.go b/go/internal/llm/client.go index e93d3f34..294f04da 100644 --- a/go/internal/llm/client.go +++ b/go/internal/llm/client.go @@ -1,7 +1,9 @@ // Package llm provides an HTTP client for LLM inference. -// Supports two wire protocols via the "protocol" parameter: +// Supports three wire protocols via the backend's Protocol field: // - "ollama": Ollama-native /api/chat (supports think, num_ctx) // - "openai": OpenAI-compatible /v1/chat/completions (works with any provider) +// - "anthropic": native Anthropic Messages API /v1/messages (anthropic.go: +// cached system prompt, output_config.effort, no sampling parameters) package llm import ( @@ -104,13 +106,21 @@ type Options struct { // provider actually said. length is the output-cap signal the dream pipeline // reads to tell a truncated answer from a malformed one. type ChatResponse struct { - Message Message - EvalCount int - PromptTokens int - FinishReason string - CostUSD *float64 - ServedModel string - ProviderRequestID string + Message Message + EvalCount int + PromptTokens int + // CacheReadTokens / CacheCreationTokens are the prompt-cache counters of + // the anthropic wire (usage.cache_read_input_tokens / + // cache_creation_input_tokens). They are ALREADY INCLUDED in + // PromptTokens — that field is the total prompt on every wire — and + // travel separately only so llmlog can record the cache hit rate + // (ApplyProviderTelemetry → metadata). Zero on every other wire. + CacheReadTokens int + CacheCreationTokens int + FinishReason string + CostUSD *float64 + ServedModel string + ProviderRequestID string // TopLogprobs are the provider's top-N alternatives for the FIRST // generated token (token text + natural-log probability), present only // when the request asked for them (Options.TopLogprobs > 0) AND the @@ -241,10 +251,14 @@ func ChatJSON(ctx context.Context, b backends.Backend, systemPrompt, userPrompt // ExtraBody are wire-relevant (OpenRouter forced zdr/deny, per-backend // attribution headers), so the loose-parameter form would silently drop them. func chatWithFormat(ctx context.Context, b backends.Backend, systemPrompt, userPrompt string, opts Options, format string, timeout time.Duration) (*ChatResponse, error) { - if b.Protocol == backends.ProtocolOpenAI { + switch b.Protocol { + case backends.ProtocolOpenAI: return chatOpenAI(ctx, b, systemPrompt, userPrompt, opts, format, timeout) + case backends.ProtocolAnthropic: + return chatAnthropic(ctx, b, systemPrompt, userPrompt, opts, format, timeout) + default: + return chatOllama(ctx, b, systemPrompt, userPrompt, opts, format, timeout) } - return chatOllama(ctx, b, systemPrompt, userPrompt, opts, format, timeout) } func chatOllama(ctx context.Context, b backends.Backend, systemPrompt, userPrompt string, opts Options, format string, timeout time.Duration) (*ChatResponse, error) { diff --git a/go/internal/llm/finishreason_wire_test.go b/go/internal/llm/finishreason_wire_test.go index 1b63ca1a..f9d11ff3 100644 --- a/go/internal/llm/finishreason_wire_test.go +++ b/go/internal/llm/finishreason_wire_test.go @@ -40,6 +40,12 @@ func TestChatDecodesFinishReason(t *testing.T) { "usage":{"completion_tokens":600,"prompt_tokens":120}}` openAINoReason = `{"choices":[{"message":{"role":"assistant","content":"hi"}}], "usage":{"completion_tokens":600,"prompt_tokens":120}}` + // The anthropic wire MAPS its vocabulary (max_tokens → length) so the + // cap consumers see it; absence still decodes to "" (anthropic.go). + anthropicWithReason = `{"id":"msg_1","content":[{"type":"text","text":"hi"}], + "stop_reason":"max_tokens","usage":{"input_tokens":120,"output_tokens":600}}` + anthropicNoReason = `{"id":"msg_1","content":[{"type":"text","text":"hi"}], + "usage":{"input_tokens":120,"output_tokens":600}}` ) tests := []struct { @@ -52,6 +58,8 @@ func TestChatDecodesFinishReason(t *testing.T) { {"ollama-absent", backends.ProtocolOllama, ollamaNoReason, ""}, {"openai-length", backends.ProtocolOpenAI, openAIWithReason, "length"}, {"openai-absent", backends.ProtocolOpenAI, openAINoReason, ""}, + {"anthropic-max_tokens", backends.ProtocolAnthropic, anthropicWithReason, "length"}, + {"anthropic-absent", backends.ProtocolAnthropic, anthropicNoReason, ""}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/go/internal/llm/wire_test.go b/go/internal/llm/wire_test.go index 834d389c..488bd635 100644 --- a/go/internal/llm/wire_test.go +++ b/go/internal/llm/wire_test.go @@ -42,6 +42,10 @@ func newWireRecorder(t *testing.T, content string) *wireRecorder { fmt.Fprintf(w, `{"choices":[{"message":{"role":"assistant","content":%q}}],"usage":{"completion_tokens":1,"prompt_tokens":1}}`, content) return } + if r.URL.Path == "/v1/messages" { + fmt.Fprintf(w, `{"id":"msg_1","content":[{"type":"text","text":%q}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`, content) + return + } fmt.Fprintf(w, `{"message":{"role":"assistant","content":%q},"eval_count":1,"prompt_eval_count":1}`, content) })) t.Cleanup(rec.srv.Close) @@ -70,6 +74,8 @@ var wireCases = []struct { }{ {"ollama", backends.ProtocolOllama, "/api/chat", `"options"`}, {"openai", backends.ProtocolOpenAI, "/v1/chat/completions", `"max_tokens"`}, + // anthropic: output_config is a Messages-API-only field. + {"anthropic", backends.ProtocolAnthropic, "/v1/messages", `"output_config"`}, } func assertWire(t *testing.T, rec *wireRecorder, wantPath, bodyMark string) { @@ -167,6 +173,9 @@ func TestChatJSONWirePath(t *testing.T) { jsonMarks := map[backends.Protocol]string{ backends.ProtocolOllama: `"format":"json"`, backends.ProtocolOpenAI: `"response_format"`, + // No JSON-mode marker on the anthropic wire (documented no-op — + // output_config.format needs a schema); the path is the assertion. + backends.ProtocolAnthropic: `"max_tokens"`, } for _, tc := range wireCases { t.Run(tc.name, func(t *testing.T) { From 48b50e3874588f762e3ca0085a5f2cd8b45fc8eb Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 00:17:52 +0200 Subject: [PATCH 06/16] feat(llm): Anthropic SSE streaming with tool use for the chat harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ChatStreamBackend is the new protocol dispatch of the streaming path: it takes the Backend (ChatStream took host/key/model loosely and never read b.Protocol) and routes ProtocolAnthropic to chatStreamAnthropic (anthropic_stream.go), everything else to ChatStream as before — ollama rows included, which always streamed over /v1/chat/completions. The chat harness (chat/engine.go runStream) calls it; ChatStream stays exported as the openai implementation its tests drive. The anthropic leg feeds the same StreamEvent/StreamResult contract: a dedicated SSE parser keys on the data frame's own "type" (message_start → prompt + cache counters; content_block_start tool_use → pending call and the ToolCallName event; text_delta → ContentDelta; input_json_delta → arguments fragments, assembled into the object form ToolCall carries; message_delta → mapped stop reason and output_tokens; message_stop → done; error → *StreamError with the provider's error type; ping, thinking_delta and signature_delta ignored). Lenient ending as on the openai parser: EOF after message_delta is a complete turn, EOF before it is a hard error. Message mapping: system rows lift into the cached top-level system block, an assistant row with tool calls becomes [text?, tool_use…] (unparseable arguments replay as {} — the executor already answered them ok:false), a tool row becomes a user turn with a tool_result block, and consecutive same-role turns fold into one message (the API requires alternation; a parallel-call turn yields several tool rows). ToolDef becomes {name, description, input_schema}; no tool_choice (a 400 on Claude Opus 5.5; auto is what the openai wire sends too). StreamResult gains the cache counters and the message id; recordLLM writes them into the web-chat llmlog row (metadata.cache_read_input_tokens / cache_creation_input_tokens / provider_request_id) through the shared ApplyCacheTelemetry. Tests: three hand-written fixtures from the Messages API event reference (content with thinking + ping, two tool calls with fragmented partial_json and an empty-input call, mid-stream overloaded_error); lenient and hard endings; 529 → ClassGateway via errors.As; callback abort; the full tool round-trip request shape (headers, cached system, folded tool results, input_schema, effort, forbidden keys absent); the dispatch across openai/ollama/anthropic rows; message edge cases. The MW6 dispatch invariant drives the eighth wire site (llm/anthropic_stream.go pinned, Σ acquires == Σ wire contacts holds at 8/8 and 10/10). docs/api.md: the web-chat harness paragraph names the per-protocol stream. Co-Authored-By: Claude Fable 5.1 --- docs/api.md | 2 +- go/internal/chat/dispatch_invariant_test.go | 39 +- go/internal/chat/engine.go | 7 +- go/internal/llm/anthropic_stream.go | 389 ++++++++++++++++ go/internal/llm/anthropic_stream_test.go | 415 ++++++++++++++++++ go/internal/llm/stream.go | 27 +- .../llm/testdata/anthropic_content.sse | 36 ++ .../testdata/anthropic_midstream_error.sse | 12 + .../llm/testdata/anthropic_tool_use.sse | 39 ++ 9 files changed, 951 insertions(+), 15 deletions(-) create mode 100644 go/internal/llm/anthropic_stream.go create mode 100644 go/internal/llm/anthropic_stream_test.go create mode 100644 go/internal/llm/testdata/anthropic_content.sse create mode 100644 go/internal/llm/testdata/anthropic_midstream_error.sse create mode 100644 go/internal/llm/testdata/anthropic_tool_use.sse diff --git a/docs/api.md b/docs/api.md index 23060fae..78728307 100644 --- a/docs/api.md +++ b/docs/api.md @@ -449,7 +449,7 @@ message — the verdict lives in the body, like every `backend-test` outcome. (F6, migration 056) The persistence layer, server-side tool harness and streaming HTTP endpoint for web chat. `context_chat_sessions` is **scope-owned**: list and delete key on the creating tenant's home scope, so a key never sees a foreign tenant's chats. It snapshots the creating key's `read_scopes` and carries a **monotone `max_sensitivity` high-water-mark**. Reading or continuing a session requires `session.read_scopes ⊆ caller.ReadScopes` (else 404, no oracle), closing the shadow-corpus channel against future least-privilege keys. The HWM rises with **every** appended message (in the same short transaction that assigns the message `seq`), so a credentials-touched session stays full-trust-only for its whole life. `context_chat_messages` records per-message `sensitivity` (fail-closed `credentials` default), tool-call metadata, telemetry and a gapless `seq` (`UNIQUE(session_id, seq)`). A turn claims its session via a short `busy_until` CAS — a second concurrent turn gets 409 without blocking, a crashed turn self-heals on expiry. Retention is off by default (`CTX_WEBCHAT_SESSION_RETENTION`). -The **harness** (`internal/chat`) drives the model loop (model call → tool execution → next call), re-resolving the F3 chat chain each iteration on `max(request, session HWM)` sensitivity (an empty chain ends the turn — never a silent escalation). Four read-only tools run under the session's `read_scopes` snapshot: **ctx_query** (hybrid retrieval, delegated to `/api/query` with `synthesize:false`+`include_content:true`, attributed to the real key), **ctx_search**, **ctx_get** (full block, paged past the window via a resumable `offset`), **ctx_recent**. Each result is annotated with `max(sensitivity)` of the blocks it carried; tools are offered only to a full-trust backend, and the closing call after the tool-budget cap carries no tools array. Tool errors return to the model as `{"error":…}` and never abort the turn. Events flow through a narrow `Sink` interface, so a future headless agent runner can drive the same loop without HTTP. +The **harness** (`internal/chat`) drives the model loop (model call → tool execution → next call), re-resolving the F3 chat chain each iteration on `max(request, session HWM)` sensitivity (an empty chain ends the turn — never a silent escalation). Each model call streams over the wire of the backend's `protocol` (`ChatStreamBackend`): the OpenAI-compatible SSE for `openai` and `ollama` rows, the native Messages API SSE for `anthropic` rows (tool calls as `tool_use`/`tool_result` blocks, see the Anthropic paragraph under [Backend pool](#backend-pool)) — the harness's event contract (`delta`, `tool_call_start`, `done`) is the same on both. Four read-only tools run under the session's `read_scopes` snapshot: **ctx_query** (hybrid retrieval, delegated to `/api/query` with `synthesize:false`+`include_content:true`, attributed to the real key), **ctx_search**, **ctx_get** (full block, paged past the window via a resumable `offset`), **ctx_recent**. Each result is annotated with `max(sensitivity)` of the blocks it carried; tools are offered only to a full-trust backend, and the closing call after the tool-budget cap carries no tools array. Tool errors return to the model as `{"error":…}` and never abort the turn. Events flow through a narrow `Sink` interface, so a future headless agent runner can drive the same loop without HTTP. **Endpoints** (auth required; `CTX_WEBCHAT_ENABLED=false` ⇒ 404): diff --git a/go/internal/chat/dispatch_invariant_test.go b/go/internal/chat/dispatch_invariant_test.go index 796b166c..f6424d36 100644 --- a/go/internal/chat/dispatch_invariant_test.go +++ b/go/internal/chat/dispatch_invariant_test.go @@ -125,7 +125,15 @@ func newWireServer(t *testing.T, wireHits *int64) *httptest.Server { fmt.Fprintf(w, `{"embeddings":[[%s]],"prompt_eval_count":7}`, vec) case "/v1/embeddings": // embed.go:185 (openai) fmt.Fprintf(w, `{"data":[{"embedding":[%s]}]}`, vec) - case "/v1/messages": // anthropic.go (native Messages API, non-stream) + case "/v1/messages": // anthropic.go (non-stream) OR anthropic_stream.go + if bytes.Contains(body, []byte(`"stream":true`)) { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, "data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":10}}}\n\n"+ + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ok\"}}\n\n"+ + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":5}}\n\n"+ + "data: {\"type\":\"message_stop\"}\n\n") + return + } fmt.Fprint(w, `{"id":"msg_1","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":10,"output_tokens":5}}`) case "/v1/rerank": // rerank.go:96 var req struct { @@ -221,9 +229,17 @@ func TestDispatchAcquireEqualsWireAllSites(t *testing.T) { t.Fatalf("anthropic non-stream: %v", err) } - // Seven sites, each one physical wire contact under one lease. - if ca.count() != 7 || atomic.LoadInt64(&wireHits) != 7 { - t.Fatalf("after the seven sites: acquires=%d wire=%d, want 7/7", ca.count(), atomic.LoadInt64(&wireHits)) + // --- Site 8: anthropic_stream.go — the ChatStreamBackend dispatch under + // the same stream lease as site 3. --- + so = e.runStream(ctx, []backends.Backend{invariantBackend(srv.URL, backends.ProtocolAnthropic)}, + []llm.ChatMsg{{Role: "user", Content: "hi"}}, false, 0, 1, "sess", "key", backends.SensPublic, llmAdm, noopSink{}) + if !so.served || so.err != nil { + t.Fatalf("anthropic stream: served=%v err=%v", so.served, so.err) + } + + // Eight sites, each one physical wire contact under one lease. + if ca.count() != 8 || atomic.LoadInt64(&wireHits) != 8 { + t.Fatalf("after the eight sites: acquires=%d wire=%d, want 8/8", ca.count(), atomic.LoadInt64(&wireHits)) } // --- Failover walk (design W5: "inkl. Failover-Pfade"): a down primary @@ -240,8 +256,8 @@ func TestDispatchAcquireEqualsWireAllSites(t *testing.T) { "chat", "s", "u", llm.Options{}, "", time.Second, nil, llmAdm); err != nil || served == nil || len(attempts) != 2 { t.Fatalf("failover walk: served=%v attempts=%d err=%v", served, len(attempts), err) } - if ca.count() != 9 || atomic.LoadInt64(&wireHits) != 9 { - t.Fatalf("after failover: acquires=%d wire=%d, want 9/9", ca.count(), atomic.LoadInt64(&wireHits)) + if ca.count() != 10 || atomic.LoadInt64(&wireHits) != 10 { + t.Fatalf("after failover: acquires=%d wire=%d, want 10/10", ca.count(), atomic.LoadInt64(&wireHits)) } // --- Lease-free without wire contact (the cache-hit clause's structural @@ -270,11 +286,12 @@ func TestDispatchAcquireEqualsWireAllSites(t *testing.T) { // deliberately added/removed AND its acquire coupling is proven in // TestDispatchAcquireEqualsWireAllSites. var wirePin = map[string]int{ - "llm/client.go": 2, // /api/chat + /v1/chat/completions (non-stream) - "llm/stream.go": 1, // /v1/chat/completions (stream) - "llm/anthropic.go": 1, // /v1/messages (anthropic non-stream) - "embed/embed.go": 2, // /api/embed + /v1/embeddings - "rerank/rerank.go": 1, // /v1/rerank + "llm/client.go": 2, // /api/chat + /v1/chat/completions (non-stream) + "llm/stream.go": 1, // /v1/chat/completions (stream) + "llm/anthropic.go": 1, // /v1/messages (anthropic non-stream) + "llm/anthropic_stream.go": 1, // /v1/messages (anthropic stream) + "embed/embed.go": 2, // /api/embed + /v1/embeddings + "rerank/rerank.go": 1, // /v1/rerank } // TestWireSitePositiveListPinned pins the grep positive list: the ONLY diff --git a/go/internal/chat/engine.go b/go/internal/chat/engine.go index 745577a9..2f2b1c2f 100644 --- a/go/internal/chat/engine.go +++ b/go/internal/chat/engine.go @@ -426,7 +426,7 @@ func (e *Engine) runStream(ctx context.Context, chain []backends.Backend, msgs [ // fires after ChatStream drained the stream, so the lease spans // first byte to stream end. defer lease.Release() - res, err := llm.ChatStream(runCtx, b.Host, b.APIKey, model, msgs, tools, opts, b.ExtraBody, e.cfg.LLMTimeout, + res, err := llm.ChatStreamBackend(runCtx, b, model, msgs, tools, opts, e.cfg.LLMTimeout, func(ev llm.StreamEvent) error { started = true if ev.ContentDelta != "" { @@ -584,6 +584,11 @@ func (e *Engine) recordLLM(sessID, apiKeyID string, required backends.Sensitivit entry.PromptTokens = res.PromptTokens entry.CompletionTokens = res.CompletionTokens entry.Metadata["tool_calls"] = len(res.ToolCalls) + if res.ProviderRequestID != "" { + entry.Metadata["provider_request_id"] = res.ProviderRequestID + } + // anthropic wire: prompt-cache counters (no-op on the openai wire). + llm.ApplyCacheTelemetry(&entry, res.CacheReadTokens, res.CacheCreationTokens) } recordEntry(e.pool, entry) } diff --git a/go/internal/llm/anthropic_stream.go b/go/internal/llm/anthropic_stream.go new file mode 100644 index 00000000..e84a5a74 --- /dev/null +++ b/go/internal/llm/anthropic_stream.go @@ -0,0 +1,389 @@ +// Streaming chat with tool use over the native Anthropic Messages API +// (POST /v1/messages, stream:true) — the ProtocolAnthropic leg of +// ChatStreamBackend, feeding the SAME StreamEvent/StreamResult contract the +// web-chat harness (chat/engine.go) consumes from the openai wire. +// +// Wire shape (claude-api reference, 2026-10): every data frame carries its +// own "type", so the parser keys on that and ignores the SSE event: lines +// (same information, and keying on the payload survives a proxy that drops +// the event field). Frames handled: +// - message_start → usage (input + cache counters; the prompt total) +// - content_block_start→ tool_use: new pending call, ToolCallName event; +// text/thinking: nothing +// - content_block_delta→ text_delta: ContentDelta; input_json_delta: +// arguments fragment; thinking_delta / +// signature_delta: ignored (empty under the +// default display anyway) +// - message_delta → stop_reason (mapped, FinishReason event) and +// usage.output_tokens +// - message_stop → done +// - error → *StreamError (HTTP 200, mid-stream — the same +// class OpenRouter reports) +// - ping → ignored +// +// Message mapping ([]ChatMsg → Messages API): system rows lift into the +// cached top-level system block; an assistant row with tool calls becomes +// [text?, tool_use…] blocks; a tool row becomes a user turn carrying a +// tool_result block, and consecutive same-role turns fold into ONE message +// (the API requires user/assistant alternation — the harness appends one +// tool row per call, so a parallel-call turn yields several tool rows in +// a row). ToolDef (OpenAI function envelope) becomes {name, description, +// input_schema}. tool_choice is never sent (forced choice is a 400 on +// Claude Opus 5.5; auto is also what buildStreamBody sends). +package llm + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/httpx" +) + +// --- request mapping ---. + +// anthropicToolUseBlock / anthropicToolResultBlock / anthropicTextBlock are +// the content blocks of the chat harness's turns. +type anthropicToolUseBlock struct { + Type string `json:"type"` // "tool_use" + ID string `json:"id"` + Name string `json:"name"` + Input json.RawMessage `json:"input"` +} + +type anthropicToolResultBlock struct { + Type string `json:"type"` // "tool_result" + ToolUseID string `json:"tool_use_id"` + Content string `json:"content"` +} + +// anthropicMessages converts the multi-turn history: the joined system text +// and the alternating message list. Empty assistant turns without tool +// calls are dropped (the API rejects empty content), and an assistant turn +// that would end the list is kept — the harness never ends on one (it +// appends tool results), and a trailing assistant turn is a prefill the +// Claude 5 models reject with a clear 400 rather than something to mask. +func anthropicMessages(msgs []ChatMsg) (system string, out []anthropicMessage) { + var sys []string + appendBlocks := func(role string, blocks ...any) { + if n := len(out); n > 0 && out[n-1].Role == role { + existing, _ := out[n-1].Content.([]any) + out[n-1].Content = append(existing, blocks...) + return + } + out = append(out, anthropicMessage{Role: role, Content: blocks}) + } + for _, m := range msgs { + switch m.Role { + case "system": + if m.Content != "" { + sys = append(sys, m.Content) + } + case "user": + appendBlocks("user", anthropicTextBlock{Type: "text", Text: m.Content}) + case "assistant": + var blocks []any + if m.Content != "" { + blocks = append(blocks, anthropicTextBlock{Type: "text", Text: m.Content}) + } + for _, tc := range m.ToolCalls { + blocks = append(blocks, anthropicToolUseBlock{ + Type: "tool_use", ID: tc.ID, Name: tc.Function.Name, + Input: anthropicToolInput(tc.Function.Arguments), + }) + } + if len(blocks) == 0 { + continue + } + appendBlocks("assistant", blocks...) + case "tool": + appendBlocks("user", anthropicToolResultBlock{Type: "tool_result", ToolUseID: m.ToolCallID, Content: m.Content}) + } + } + return strings.Join(sys, "\n\n"), out +} + +// anthropicToolInput renders assembled tool-call arguments as the tool_use +// input object. ToolCall carries the object form already (normalizeArgs); +// anything that is not a valid JSON object becomes {} — the executor has +// already answered such a call with an ok:false result, and the history +// must still replay. +func anthropicToolInput(args json.RawMessage) json.RawMessage { + t := bytes.TrimSpace(args) + if len(t) > 0 && t[0] == '{' && json.Valid(t) { + return t + } + return json.RawMessage("{}") +} + +// anthropicTools converts the OpenAI function envelopes. input_schema is +// required; a tool without parameters gets the empty object schema. +func anthropicTools(tools []ToolDef) []anthropicTool { + if len(tools) == 0 { + return nil + } + out := make([]anthropicTool, 0, len(tools)) + for _, t := range tools { + schema := bytes.TrimSpace(t.Function.Parameters) + if len(schema) == 0 || bytes.Equal(schema, []byte("null")) { + schema = []byte(`{"type":"object","properties":{}}`) + } + out = append(out, anthropicTool{Name: t.Function.Name, Description: t.Function.Description, InputSchema: schema}) + } + return out +} + +// chatStreamAnthropic is the streaming twin of chatAnthropic. Same retry +// doctrine as ChatStream: httpx.DoRetryOnce covers connection setup only — +// nothing is replayed once response bytes arrived. +func chatStreamAnthropic(ctx context.Context, b backends.Backend, model string, + msgs []ChatMsg, tools []ToolDef, opts Options, timeout time.Duration, + onEvent func(StreamEvent) error) (*StreamResult, error) { + system, messages := anthropicMessages(msgs) + req := anthropicRequest{ + Model: model, + MaxTokens: anthropicMaxTokens(opts), + System: anthropicSystem(system), + Messages: messages, + Stream: true, + OutputConfig: anthropicOutputConfig{Effort: resolveAnthropicEffort(opts, &b)}, + Tools: anthropicTools(tools), + } + body, err := buildAnthropicBody(req, opts, &b) + if err != nil { + return nil, fmt.Errorf("llm: marshal request: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, b.Host+"/v1/messages", bytes.NewReader(body)) //nolint:gosec // G704: host from the F3 backend pool (context_backends) — admin-gated + locality-validated, no free user input. + if err != nil { + return nil, fmt.Errorf("llm: create request: %w", err) + } + setAnthropicHeaders(httpReq, &b) + httpReq.Header.Set("Accept", "text/event-stream") + + resp, err := httpx.DoRetryOnce(httpClient, httpReq, body) + if err != nil { + return nil, fmt.Errorf("llm: request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + // Typed StatusError for the failover classifier, as on every wire. + return nil, anthropicStatusError(resp) + } + + return parseAnthropicSSE(resp.Body, onEvent) +} + +// --- SSE parsing ---. + +// anthropicStreamFrame is the union of every data frame; only the fields +// the parser reads are modelled. +type anthropicStreamFrame struct { + Type string `json:"type"` + Index int `json:"index"` + Message *struct { + ID string `json:"id"` + Usage anthropicUsage `json:"usage"` + } `json:"message"` + ContentBlock *struct { + Type string `json:"type"` + ID string `json:"id"` + Name string `json:"name"` + } `json:"content_block"` + Delta *struct { + Type string `json:"type"` + Text string `json:"text"` + PartialJSON string `json:"partial_json"` + StopReason string `json:"stop_reason"` + } `json:"delta"` + Usage *anthropicUsage `json:"usage"` + Error *struct { + Type string `json:"type"` + Message string `json:"message"` + } `json:"error"` +} + +// anthropicStreamState carries the assembly across frames. +type anthropicStreamState struct { + onEvent func(StreamEvent) error + content strings.Builder + calls []*pendingToolCall + byIndex map[int]*pendingToolCall + finish string + usage anthropicUsage + stopSeen bool + done bool + messageID string + outputSeen bool +} + +func (st *anthropicStreamState) emit(ev StreamEvent) error { + if st.onEvent == nil { + return nil + } + if err := st.onEvent(ev); err != nil { + return fmt.Errorf("llm: stream event callback: %w", err) + } + return nil +} + +// parseAnthropicSSE reads frames until message_stop or EOF. Lenient ending +// mirrors parseSSEStream: EOF after message_delta (stop reason known) is a +// complete turn; EOF before it is a hard error — a mid-stream connection +// death must not look like a finished answer. +func parseAnthropicSSE(r io.Reader, onEvent func(StreamEvent) error) (*StreamResult, error) { + st := &anthropicStreamState{onEvent: onEvent, byIndex: map[int]*pendingToolCall{}} + br := bufio.NewReader(r) + var data []string + for { + line, readErr := br.ReadString('\n') + if err := st.feedLine(strings.TrimRight(line, "\r\n"), &data); err != nil { + return nil, err + } + if st.done { + break + } + if readErr != nil { + if errors.Is(readErr, io.EOF) { + if err := st.dispatchData(&data); err != nil { + return nil, err + } + break + } + return nil, fmt.Errorf("llm: read stream: %w", readErr) + } + } + if !st.done && !st.stopSeen { + return nil, fmt.Errorf("llm: stream ended unexpectedly (no message_stop, no message_delta stop_reason)") + } + return st.result(), nil +} + +// feedLine: blank → dispatch the buffered frame; ":" → comment; "data:" → +// buffer; "event:"/"id:"/"retry:" → ignored (the type travels in the data). +func (st *anthropicStreamState) feedLine(line string, data *[]string) error { + switch { + case line == "": + return st.dispatchData(data) + case strings.HasPrefix(line, ":"): + return nil + case strings.HasPrefix(line, "data:"): + v := strings.TrimPrefix(line, "data:") + v = strings.TrimPrefix(v, " ") + *data = append(*data, v) + return nil + default: + return nil + } +} + +func (st *anthropicStreamState) dispatchData(data *[]string) error { + if len(*data) == 0 { + return nil + } + payload := strings.Join(*data, "\n") + *data = (*data)[:0] + return st.handleFrame(payload) +} + +func (st *anthropicStreamState) handleFrame(payload string) error { + var f anthropicStreamFrame + if err := json.Unmarshal([]byte(payload), &f); err != nil { + return fmt.Errorf("llm: malformed stream chunk: %w", err) + } + switch f.Type { + case "error": + if f.Error == nil { + return &StreamError{Message: "stream error event without an error object"} + } + return &StreamError{Code: f.Error.Type, Message: f.Error.Message} + case "message_start": + if f.Message != nil { + st.messageID = f.Message.ID + st.usage = f.Message.Usage + } + case "content_block_start": + if f.ContentBlock != nil && f.ContentBlock.Type == "tool_use" { + pc := &pendingToolCall{index: f.Index, id: f.ContentBlock.ID, typ: "function", name: f.ContentBlock.Name} + st.byIndex[f.Index] = pc + st.calls = append(st.calls, pc) + if pc.name != "" { + pc.nameEmitted = true + return st.emit(StreamEvent{ToolCallName: pc.name}) + } + } + case "content_block_delta": + if f.Delta == nil { + return nil + } + switch f.Delta.Type { + case "text_delta": + if f.Delta.Text != "" { + st.content.WriteString(f.Delta.Text) + return st.emit(StreamEvent{ContentDelta: f.Delta.Text}) + } + case "input_json_delta": + if pc := st.byIndex[f.Index]; pc != nil { + pc.args.WriteString(f.Delta.PartialJSON) + } + } + // thinking_delta / signature_delta: never surfaced. + case "message_delta": + if f.Usage != nil { + st.outputSeen = true + st.usage.OutputTokens = f.Usage.OutputTokens + // Newer servers repeat the (cumulative) input counters here; + // take them when present so the total is the final one. + if f.Usage.InputTokens > 0 || f.Usage.CacheReadInputTokens > 0 || f.Usage.CacheCreationInputTokens > 0 { + st.usage.InputTokens = f.Usage.InputTokens + st.usage.CacheReadInputTokens = f.Usage.CacheReadInputTokens + st.usage.CacheCreationInputTokens = f.Usage.CacheCreationInputTokens + } + } + if f.Delta != nil && f.Delta.StopReason != "" && st.finish == "" { + st.stopSeen = true + st.finish = mapAnthropicStopReason(f.Delta.StopReason) + return st.emit(StreamEvent{FinishReason: st.finish}) + } + case "message_stop": + st.done = true + } + return nil +} + +func (st *anthropicStreamState) result() *StreamResult { + prompt, output, cacheRead, cacheCreate := anthropicUsageTotals(st.usage) + res := &StreamResult{ + Content: st.content.String(), + FinishReason: st.finish, + PromptTokens: prompt, + CompletionTokens: output, + CacheReadTokens: cacheRead, + CacheCreationTokens: cacheCreate, + ProviderRequestID: st.messageID, + } + for _, pc := range st.calls { + args := bytes.TrimSpace(pc.args.Bytes()) + if len(args) == 0 { + args = []byte("{}") + } + res.ToolCalls = append(res.ToolCalls, ToolCall{ + ID: pc.id, + Type: "function", + Function: ToolCallFunction{Name: pc.name, Arguments: args}, + }) + } + return res +} diff --git a/go/internal/llm/anthropic_stream_test.go b/go/internal/llm/anthropic_stream_test.go new file mode 100644 index 00000000..738537aa --- /dev/null +++ b/go/internal/llm/anthropic_stream_test.go @@ -0,0 +1,415 @@ +// Native Anthropic Messages API wire (anthropic_stream.go), streaming path: +// fixture parsing (text, thinking, tool use, mid-stream error), lenient +// and hard endings, HTTP status errors, the request shape of a multi-turn +// tool round trip, and the ChatStreamBackend dispatch. Fixtures are +// hand-written from the Messages API event reference (claude-api, 2026-10), +// not live captures — what they cannot prove is live-only (cache hits, +// effort effect, 529 behaviour). +package llm + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/httpx" +) + +func anthropicStreamBackend(host string) backends.Backend { + b := anthropicTestBackend(host) + b.ModelMap = map[string]backends.ModelSpec{"default": {Model: "claude-opus-5-5"}} + return b +} + +func TestAnthropicStreamContentFixture(t *testing.T) { + srv := serveFixture(t, "anthropic_content.sse") + defer srv.Close() + var events []StreamEvent + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "claude-opus-5-5", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 64}, 5*time.Second, collectEvents(&events)) + if err != nil { + t.Fatalf("ChatStreamBackend: %v", err) + } + if res.Content != "Berlin is sunny." { + t.Errorf("content = %q (thinking deltas must never leak)", res.Content) + } + if res.FinishReason != "stop" { + t.Errorf("finish = %q, want end_turn mapped to stop", res.FinishReason) + } + if res.PromptTokens != 312 || res.CompletionTokens != 9 { + t.Errorf("tokens = %d/%d, want 12+300 prompt (total incl. cache read) and 9 output", res.PromptTokens, res.CompletionTokens) + } + if res.CacheReadTokens != 300 || res.CacheCreationTokens != 0 { + t.Errorf("cache counters = %d/%d, want 300/0", res.CacheReadTokens, res.CacheCreationTokens) + } + if res.ProviderRequestID != "msg_01stream" { + t.Errorf("ProviderRequestID = %q", res.ProviderRequestID) + } + if len(res.ToolCalls) != 0 { + t.Errorf("tool calls = %v, want none", res.ToolCalls) + } + var joined strings.Builder + finishes := 0 + for i, ev := range events { + joined.WriteString(ev.ContentDelta) + if ev.ToolCallName != "" { + t.Errorf("unexpected tool event %+v", ev) + } + if ev.FinishReason != "" { + finishes++ + if i != len(events)-1 { + t.Errorf("finish event at %d of %d, want last", i, len(events)) + } + } + } + if joined.String() != res.Content || finishes != 1 { + t.Errorf("deltas = %q finishes = %d", joined.String(), finishes) + } +} + +func TestAnthropicStreamToolUseFixture(t *testing.T) { + srv := serveFixture(t, "anthropic_tool_use.sse") + defer srv.Close() + var events []StreamEvent + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "claude-opus-5-5", + []ChatMsg{{Role: "user", Content: "weather?"}}, nil, Options{NumPredict: 64}, 5*time.Second, collectEvents(&events)) + if err != nil { + t.Fatalf("ChatStreamBackend: %v", err) + } + if res.FinishReason != "tool_calls" { + t.Errorf("finish = %q, want tool_use mapped to tool_calls (the harness's dispatch key)", res.FinishReason) + } + if res.Content != "Let me check." { + t.Errorf("content = %q", res.Content) + } + if len(res.ToolCalls) != 2 { + t.Fatalf("tool calls = %d, want 2", len(res.ToolCalls)) + } + first := res.ToolCalls[0] + if first.ID != "toolu_01weather" || first.Type != "function" || first.Function.Name != "get_weather" { + t.Errorf("first call = %+v", first) + } + args := argsAsMap(t, first) + if args["city"] != "Berlin" || args["unit"] != "c" { + t.Errorf("arguments = %v, want the partial_json fragments joined into one object", args) + } + second := res.ToolCalls[1] + if second.ID != "toolu_02time" || second.Function.Name != "get_time" || string(second.Function.Arguments) != "{}" { + t.Errorf("second call = %+v, want empty arguments as {}", second) + } + if res.PromptTokens != 552 || res.CompletionTokens != 31 || res.CacheCreationTokens != 512 { + t.Errorf("tokens = %d/%d cache_create=%d, want 40+512 / 31 / 512", res.PromptTokens, res.CompletionTokens, res.CacheCreationTokens) + } + var names []string + for _, ev := range events { + if ev.ToolCallName != "" { + names = append(names, ev.ToolCallName) + } + } + if strings.Join(names, ",") != "get_weather,get_time" { + t.Errorf("tool name events = %v, want one per call in stream order", names) + } + if last := events[len(events)-1]; last.FinishReason != "tool_calls" { + t.Errorf("last event = %+v, want the finish", last) + } + + // The assembled calls round-trip into the next request as tool_use + // blocks with the SAME ids, so tool_result blocks can reference them. + system, msgs := anthropicMessages([]ChatMsg{ + {Role: "system", Content: "sys"}, + {Role: "user", Content: "weather?"}, + {Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls}, + {Role: "tool", Content: "21C", ToolCallID: "toolu_01weather"}, + {Role: "tool", Content: "12:00", ToolCallID: "toolu_02time"}, + }) + if system != "sys" || len(msgs) != 3 { + t.Fatalf("system=%q msgs=%d, want 3 turns (user, assistant, merged tool results)", system, len(msgs)) + } + raw, _ := json.Marshal(msgs) + for _, want := range []string{ + `"role":"assistant"`, `"type":"tool_use","id":"toolu_01weather","name":"get_weather","input":{"city":"Berlin","unit":"c"}`, + `"type":"tool_use","id":"toolu_02time","name":"get_time","input":{}`, + `"type":"tool_result","tool_use_id":"toolu_01weather","content":"21C"`, + `"type":"tool_result","tool_use_id":"toolu_02time","content":"12:00"`, + } { + if !strings.Contains(string(raw), want) { + t.Errorf("messages lack %s:\n%s", want, raw) + } + } + if msgs[2].Role != "user" { + t.Errorf("tool results must travel as a user turn, got %q", msgs[2].Role) + } +} + +func TestAnthropicStreamMidStreamError(t *testing.T) { + srv := serveFixture(t, "anthropic_midstream_error.sse") + defer srv.Close() + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + var se *StreamError + if !errors.As(err, &se) { + t.Fatalf("want *StreamError, got %v (res %v)", err, res) + } + if se.Code != "overloaded_error" || se.Message != "Overloaded" { + t.Errorf("StreamError = %+v", se) + } + if res != nil { + t.Errorf("result must be nil on a mid-stream error, got %+v", res) + } +} + +func TestAnthropicStreamEOFWithoutStopIsError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":1}}}\n\n" + + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"half\"}}\n\n")) + })) + defer srv.Close() + _, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + if err == nil || !strings.Contains(err.Error(), "ended unexpectedly") { + t.Errorf("err = %v, want the unexpected-end error (a dead connection must not look like a finished turn)", err) + } +} + +func TestAnthropicStreamEOFAfterMessageDeltaAccepted(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":3}}}\n\n" + + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"done\"}}\n\n" + + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"max_tokens\"},\"usage\":{\"output_tokens\":4}}")) + })) + defer srv.Close() + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + if err != nil { + t.Fatalf("EOF after message_delta must be accepted: %v", err) + } + if res.Content != "done" || res.FinishReason != "length" || res.PromptTokens != 3 || res.CompletionTokens != 4 { + t.Errorf("result = %+v", res) + } +} + +func TestAnthropicStreamHTTPErrorStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(529) + _, _ = w.Write([]byte(`{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}`)) + })) + defer srv.Close() + _, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + var se *httpx.StatusError + if !errors.As(err, &se) || se.Code != 529 { + t.Fatalf("want *httpx.StatusError 529, got %v", err) + } + if got := backends.Classify(err, backends.ProviderGeneric); got != backends.ClassGateway { + t.Errorf("Classify = %v, want Gateway (529 overloaded fails over)", got) + } +} + +func TestAnthropicStreamCallbackErrorAborts(t *testing.T) { + srv := serveFixture(t, "anthropic_content.sse") + defer srv.Close() + wantErr := errors.New("sink closed") + calls := 0 + _, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, + func(StreamEvent) error { calls++; return wantErr }) + if !errors.Is(err, wantErr) || calls != 1 { + t.Errorf("err = %v calls = %d, want the callback error after one event", err, calls) + } +} + +// TestAnthropicStreamRequestWire pins the multi-turn tool round trip on the +// wire: headers, stream flag, cached system block, user/assistant/tool +// mapping with consecutive tool results folded into ONE user turn, tool +// definitions as input_schema, effort, and the absence of every field this +// wire must not carry. +func TestAnthropicStreamRequestWire(t *testing.T) { + srv, body, hdr := captureRequestBody(t, "data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":1}}}\n\n"+ + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":1}}\n\n"+ + "data: {\"type\":\"message_stop\"}\n\n") + defer srv.Close() + + tools := []ToolDef{ + {Type: "function", Function: ToolDefFunction{ + Name: "get_weather", Description: "Weather for a city", + Parameters: json.RawMessage(`{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}`), + }}, + {Type: "function", Function: ToolDefFunction{Name: "get_time"}}, + } + msgs := []ChatMsg{ + {Role: "system", Content: "You are a weather demo assistant."}, + {Role: "user", Content: "Weather in Berlin?"}, + {Role: "assistant", Content: "Checking.", ToolCalls: []ToolCall{ + {ID: "toolu_1", Type: "function", Function: ToolCallFunction{Name: "get_weather", Arguments: json.RawMessage(`{"city":"Berlin"}`)}}, + {ID: "toolu_2", Type: "function", Function: ToolCallFunction{Name: "get_time", Arguments: json.RawMessage(`not json`)}}, + }}, + {Role: "tool", Content: `{"temperature_c":21}`, ToolCallID: "toolu_1"}, + {Role: "tool", Content: `12:00`, ToolCallID: "toolu_2"}, + } + b := anthropicStreamBackend(srv.URL) + b.ExtraBody = map[string]any{"effort": "high"} + opts := Options{Temperature: 0.7, RepeatPenalty: 1.1, NumPredict: 256} + if _, err := ChatStreamBackend(context.Background(), b, "claude-opus-5-5", msgs, tools, opts, 5*time.Second, nil); err != nil { + t.Fatalf("ChatStreamBackend: %v", err) + } + + if got := hdr.Get("x-api-key"); got != "sk-ant-test" { + t.Errorf("x-api-key = %q", got) + } + if got := hdr.Get("anthropic-version"); got != anthropicVersion { + t.Errorf("anthropic-version = %q", got) + } + if got := hdr.Get("Accept"); got != "text/event-stream" { + t.Errorf("Accept = %q", got) + } + if hdr.Get("Authorization") != "" { + t.Error("Authorization header sent on the anthropic wire") + } + + var req map[string]any + if err := json.Unmarshal(*body, &req); err != nil { + t.Fatalf("request body not JSON: %v\n%s", err, *body) + } + if req["stream"] != true || req["model"] != "claude-opus-5-5" || req["max_tokens"] != float64(256) { + t.Errorf("stream/model/max_tokens = %v/%v/%v", req["stream"], req["model"], req["max_tokens"]) + } + if oc, _ := req["output_config"].(map[string]any); oc["effort"] != "high" { + t.Errorf("output_config = %v, want extra_body effort high", req["output_config"]) + } + for _, k := range anthropicForbiddenKeys { + if v, has := req[k]; has { + t.Errorf("body carries %q = %v", k, v) + } + } + sys, _ := req["system"].([]any) + if len(sys) != 1 { + t.Fatalf("system = %v", req["system"]) + } + if blk, _ := sys[0].(map[string]any); blk["text"] != "You are a weather demo assistant." || blk["cache_control"] == nil { + t.Errorf("system block = %v, want the lifted system row with cache_control", blk) + } + + turns, _ := req["messages"].([]any) + if len(turns) != 3 { + t.Fatalf("messages = %d turns, want user / assistant / merged tool results:\n%s", len(turns), *body) + } + role := func(i int) string { m, _ := turns[i].(map[string]any); r, _ := m["role"].(string); return r } + blocks := func(i int) []any { m, _ := turns[i].(map[string]any); c, _ := m["content"].([]any); return c } + if role(0) != "user" || role(1) != "assistant" || role(2) != "user" { + t.Errorf("roles = %s/%s/%s", role(0), role(1), role(2)) + } + if ub := blocks(0); len(ub) != 1 || ub[0].(map[string]any)["text"] != "Weather in Berlin?" { + t.Errorf("user turn = %v", ub) + } + ab := blocks(1) + if len(ab) != 3 { + t.Fatalf("assistant turn = %v, want text + 2 tool_use", ab) + } + if tu, _ := ab[1].(map[string]any); tu["type"] != "tool_use" || tu["id"] != "toolu_1" || tu["name"] != "get_weather" { + t.Errorf("tool_use[0] = %v", tu) + } else if in, _ := tu["input"].(map[string]any); in["city"] != "Berlin" { + t.Errorf("tool_use[0].input = %v", tu["input"]) + } + if tu, _ := ab[2].(map[string]any); tu["id"] != "toolu_2" { + t.Errorf("tool_use[1] = %v", tu) + } else if in, _ := tu["input"].(map[string]any); in == nil || len(in) != 0 { + t.Errorf("tool_use[1].input = %v, want {} for unparseable arguments", tu["input"]) + } + tr := blocks(2) + if len(tr) != 2 { + t.Fatalf("tool-result turn = %v, want both results folded into one user turn", tr) + } + if r0, _ := tr[0].(map[string]any); r0["type"] != "tool_result" || r0["tool_use_id"] != "toolu_1" || r0["content"] != `{"temperature_c":21}` { + t.Errorf("tool_result[0] = %v", r0) + } + if r1, _ := tr[1].(map[string]any); r1["tool_use_id"] != "toolu_2" || r1["content"] != "12:00" { + t.Errorf("tool_result[1] = %v", r1) + } + for i := range turns { + if _, has := turns[i].(map[string]any)["tool_calls"]; has { + t.Errorf("turn %d carries the OpenAI tool_calls field", i) + } + } + + wt, _ := req["tools"].([]any) + if len(wt) != 2 { + t.Fatalf("tools = %v", req["tools"]) + } + if t0, _ := wt[0].(map[string]any); t0["name"] != "get_weather" || t0["description"] != "Weather for a city" || t0["input_schema"] == nil || t0["function"] != nil { + t.Errorf("tools[0] = %v, want the Messages API shape (name/description/input_schema)", t0) + } + if t1, _ := wt[1].(map[string]any); t1["input_schema"] == nil { + t.Errorf("tools[1] = %v, want an empty object schema for a parameterless tool", t1) + } +} + +// TestChatStreamBackendDispatch: the protocol field, not the host, picks +// the wire — an openai row keeps /v1/chat/completions with Bearer auth +// and its extra_body, an anthropic row takes /v1/messages. +func TestChatStreamBackendDispatch(t *testing.T) { + type hit struct{ path, auth, key string } + var hits []hit + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits = append(hits, hit{r.URL.Path, r.Header.Get("Authorization"), r.Header.Get("x-api-key")}) + if r.URL.Path == "/v1/messages" { + _, _ = w.Write([]byte("data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\ndata: {\"type\":\"message_stop\"}\n\n")) + return + } + _, _ = w.Write([]byte(minimalStopStream)) + })) + defer srv.Close() + msgs := []ChatMsg{{Role: "user", Content: "hi"}} + + oa := backends.Backend{Host: srv.URL, Protocol: backends.ProtocolOpenAI, APIKey: "sk-oa"} + if _, err := ChatStreamBackend(context.Background(), oa, "m", msgs, nil, Options{NumPredict: 4}, 5*time.Second, nil); err != nil { + t.Fatalf("openai: %v", err) + } + ol := backends.Backend{Host: srv.URL, Protocol: backends.ProtocolOllama} + if _, err := ChatStreamBackend(context.Background(), ol, "m", msgs, nil, Options{NumPredict: 4}, 5*time.Second, nil); err != nil { + t.Fatalf("ollama: %v", err) + } + if _, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", msgs, nil, Options{NumPredict: 4}, 5*time.Second, nil); err != nil { + t.Fatalf("anthropic: %v", err) + } + want := []hit{ + {"/v1/chat/completions", "Bearer sk-oa", ""}, + {"/v1/chat/completions", "", ""}, + {"/v1/messages", "", "sk-ant-test"}, + } + if len(hits) != len(want) { + t.Fatalf("hits = %v", hits) + } + for i := range want { + if hits[i] != want[i] { + t.Errorf("hit %d = %+v, want %+v", i, hits[i], want[i]) + } + } +} + +// TestAnthropicMessagesEdgeCases: consecutive user turns fold, empty +// assistant turns drop, several system rows join. +func TestAnthropicMessagesEdgeCases(t *testing.T) { + system, msgs := anthropicMessages([]ChatMsg{ + {Role: "system", Content: "a"}, + {Role: "system", Content: "b"}, + {Role: "user", Content: "one"}, + {Role: "user", Content: "two"}, + {Role: "assistant", Content: ""}, + {Role: "user", Content: "three"}, + }) + if system != "a\n\nb" { + t.Errorf("system = %q", system) + } + if len(msgs) != 1 || msgs[0].Role != "user" { + t.Fatalf("msgs = %+v, want one folded user turn (empty assistant dropped)", msgs) + } + if blocks, _ := msgs[0].Content.([]any); len(blocks) != 3 { + t.Errorf("folded user turn = %v, want three text blocks", msgs[0].Content) + } +} diff --git a/go/internal/llm/stream.go b/go/internal/llm/stream.go index 190d0acd..c73d8450 100644 --- a/go/internal/llm/stream.go +++ b/go/internal/llm/stream.go @@ -30,6 +30,7 @@ import ( "strings" "time" + "github.com/GottZ/ctx/internal/backends" "github.com/GottZ/ctx/internal/httpx" ) @@ -185,6 +186,13 @@ type StreamResult struct { PromptTokens int CompletionTokens int DraftAccept float64 // timings.draft_n_accepted/draft_n, 0 when n/a + // CacheReadTokens / CacheCreationTokens / ProviderRequestID are filled by + // the anthropic wire only (ChatResponse's twins): the cache counters are + // already inside PromptTokens and travel for llmlog metadata, the + // message id for audit. Zero on the openai wire. + CacheReadTokens int + CacheCreationTokens int + ProviderRequestID string } // StreamError is an error event delivered inside an HTTP-200 SSE stream. @@ -264,9 +272,24 @@ func buildStreamBody(model string, msgs []ChatMsg, tools []ToolDef, opts Options return json.Marshal(m) } +// ChatStreamBackend is the protocol dispatch of the streaming path: the +// anthropic wire (anthropic_stream.go) for ProtocolAnthropic, ChatStream +// (the OpenAI-compatible wire) for everything else — ollama rows included, +// which serve /v1/chat/completions too and were always streamed that way. +// The chat harness calls this; ChatStream stays exported as the openai +// implementation its tests drive directly. +func ChatStreamBackend(ctx context.Context, b backends.Backend, model string, + msgs []ChatMsg, tools []ToolDef, opts Options, timeout time.Duration, + onEvent func(StreamEvent) error) (*StreamResult, error) { + if b.Protocol == backends.ProtocolAnthropic { + return chatStreamAnthropic(ctx, b, model, msgs, tools, opts, timeout, onEvent) + } + return ChatStream(ctx, b.Host, b.APIKey, model, msgs, tools, opts, b.ExtraBody, timeout, onEvent) +} + // ChatStream sends a streaming chat request with optional tool definitions -// over the OpenAI-compatible wire (the only streaming protocol in F6) and -// assembles the response. onEvent is invoked for content fragments, tool +// over the OpenAI-compatible wire (the streaming protocol of F6; the +// anthropic wire joined via ChatStreamBackend) and assembles the response. onEvent is invoked for content fragments, tool // call starts and the finish reason; an error from onEvent aborts the // stream. Cancelling ctx aborts the in-flight generation (the HTTP request // to the backend is dropped, llama.cpp frees its slot). diff --git a/go/internal/llm/testdata/anthropic_content.sse b/go/internal/llm/testdata/anthropic_content.sse new file mode 100644 index 00000000..97900f00 --- /dev/null +++ b/go/internal/llm/testdata/anthropic_content.sse @@ -0,0 +1,36 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_01stream","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":1,"cache_read_input_tokens":300,"cache_creation_input_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"secret reasoning"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"abc"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: ping +data: {"type": "ping"} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Berlin "}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"is sunny."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":9}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/go/internal/llm/testdata/anthropic_midstream_error.sse b/go/internal/llm/testdata/anthropic_midstream_error.sse new file mode 100644 index 00000000..1a003909 --- /dev/null +++ b/go/internal/llm/testdata/anthropic_midstream_error.sse @@ -0,0 +1,12 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_01err","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":5,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"partial"}} + +event: error +data: {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}} + diff --git a/go/internal/llm/testdata/anthropic_tool_use.sse b/go/internal/llm/testdata/anthropic_tool_use.sse new file mode 100644 index 00000000..aafc6b8e --- /dev/null +++ b/go/internal/llm/testdata/anthropic_tool_use.sse @@ -0,0 +1,39 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_01tool","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":40,"output_tokens":2,"cache_read_input_tokens":0,"cache_creation_input_tokens":512}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Let me check."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01weather","name":"get_weather","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"city\": \"Ber"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"lin\", \"unit\": \"c\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_02time","name":"get_time","input":{}}} + +event: content_block_stop +data: {"type":"content_block_stop","index":2} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":31}} + +event: message_stop +data: {"type":"message_stop"} + From 02475baf6b9bf571a9e467d89ddb5b6ba2e06bbc Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 00:21:33 +0200 Subject: [PATCH 07/16] feat(backends): anthropic probe, seed, init wizard and UI enum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operator surfaces of the new wire protocol. backend-test: probeBackendURL branches on the protocol before the provider class — an anthropic row probes GET {base_url}/v1/models with x-api-key + anthropic-version (no Bearer header); the generic branch would have hit the bare root with a Bearer header, which the API answers 404 (reported as unreachable) or 401. The 1-token chat probe runs through llm.Chat as before and so takes the anthropic wire automatically. ctx backends seed: the chat leg accepts protocol anthropic and is validated on the spec, before any secret is sealed — the embed leg rejects it (the Messages API serves no embeddings), a keyless leg and a leg without num_ctx (no window discovery on this wire) fail with the reason. The same rules the server enforces as 422s, one layer earlier. ctx init: the protocol prompt names anthropic; choosing it turns the single-host wizard into a two-host one — it asks the chat model's context window (default 100000), a separate embedding host and its protocol (ollama default), and goes straight to the no-echo key prompt instead of the "does the host need a key?" detour. Web UI: 'anthropic' in the PROTOCOLS enum of the backend dialog. Tests: seed error table (anthropic on embed / keyless / no num_ctx) and the mixed-protocol happy path; the wizard flow with the scripted ask (window default, separate embed host, key sealed once, not echoed, not attached to the embed row); the handler probe against an httptest fake (path, headers, base_url/auth ok). Docs: operations.md (Backends — the two-host SeedSpec example and the spec-level rules; Models & hardware; the CTX_DREAM_JSON_MODE row's anthropic clause; the Migration 155 section with the downgrade note), README's engine-agnostic sentence. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- docs/operations.md | 20 +++++- go/internal/cli/backends_seed.go | 16 ++++- go/internal/cli/backends_seed_test.go | 23 +++++++ go/internal/cli/init.go | 46 +++++++++++-- go/internal/cli/init_test.go | 53 +++++++++++++++ go/internal/handler/backends_manage.go | 21 ++++-- .../handler/backends_test_anthropic_test.go | 68 +++++++++++++++++++ go/web/src/lib/backends.ts | 2 +- 9 files changed, 234 insertions(+), 17 deletions(-) create mode 100644 go/internal/handler/backends_test_anthropic_test.go diff --git a/README.md b/README.md index 53cd8919..f61ad37c 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Running the server (Go daemon + PostgreSQL 18 + pgvector) is `docker compose up > **Upgrading from v4? Reach v4.38.0 first.** v5.0.0 removes the `CTX_{CHAT,CHAT_FALLBACK,EMBED,DREAM,DREAM_EMBED,RERANK}_*` connection variables: backends are configured in the pool (`ctx backends`) and nowhere else. **v4.38.0 is the last release that still reads those variables**, and one boot on it carries a `.env`-only configuration into the pool for you, then names every variable and leftover settings row you still have. Going straight from an older v4 to v5 is not fatal down to v4.19.0 — the env bridge is gone by then and you re-enter your topology by hand with `ctx backends seed` — but **below v4.19.0 it is a hard stop**: v5 folds migrations 001–113 into a single baseline and refuses a database that never applied 113. Migration path and rollback notes: [operations](docs/operations.md#migration-133-the-backend-tuple-rows-are-deleted--and-the-upgrade-hop-that-has-to-come-first). -**Models:** the tested reference stack is qwen3.6:27b for chat/synthesis/dream/digest, Qwen3-Embedding-8B (1024-dim, CPU-sized) for embeddings, and a bge-reranker-v2-m3 cross-encoder sidecar for rerank — but ctx is engine-agnostic: any OpenAI-compatible or Ollama API works, models are per-backend configuration. CPU-only setups work; a GPU shortens chat latency. Details: [Models & hardware](docs/operations.md#models--hardware). +**Models:** the tested reference stack is qwen3.6:27b for chat/synthesis/dream/digest, Qwen3-Embedding-8B (1024-dim, CPU-sized) for embeddings, and a bge-reranker-v2-m3 cross-encoder sidecar for rerank — but ctx is engine-agnostic: any OpenAI-compatible or Ollama API works, and the chat-class roles can run on the native Anthropic Messages API (`protocol: "anthropic"`); models are per-backend configuration. CPU-only setups work; a GPU shortens chat latency. Details: [Models & hardware](docs/operations.md#models--hardware). ## How LLMs use ctx diff --git a/docs/operations.md b/docs/operations.md index c78180bb..e56937c3 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -114,7 +114,7 @@ The 29 names, to grep an inherited `.env` against — `CTX_CHAT_{HOST,API_KEY,PR | `CTX_DREAM_EVAL_CAP_RETRY_FACTOR` | `2` | hot | Scales the output cap of **one** retry after the dream link evaluation was truncated at that cap. Since the non-streaming chat path decodes the provider's stop reason, a cap hit is *detected* (`finish_reason` = `length`) rather than inferred; where a provider reports none, the completion-token heuristic still carries it, and `metadata.cap_hit_source` on the `context_llm_log` row names which of the two spoke. The retry re-sends the **identical** prompt — no shrinking, no candidate re-planning — so the cap is the single variable, and it writes its own log row (`metadata.cap_retry`), keeping token and cost accounting at one row per wire call. **Precedence:** the factor multiplies the *resolved* cap, i.e. after a `num_predict` / `max_tokens` param in the serving row's `model_map` (`applyModelParams`, walked in `llm/chain.go`) — a caller-side doubling would be a no-op on exactly those rows. **On a second cap hit** the block is booked as a **completed-but-inert eval** — `dream_eval_count` + 1, `dream_last_inert`, `CTX_DREAM_BACKOFF_INERT_OFFSET` on the curve — instead of the 5-minute transient cooldown. That is the point of the key: the transient path has no attempt counter, so before this a block whose prompt cannot fit re-burned one eval call every five minutes forever. The block is not abandoned; it re-dreams later on the back-off curve and links normally once `CTX_DREAM_NUM_PREDICT` or this factor is raised. **`1` or lower disables the retry** and restores the previous behaviour exactly (one call, parse error, transient cooldown, re-pick) — a true kill switch, nothing else changes with it off. **Limit:** on a serving row whose `extra_body` pins a numeric `max_tokens`, that value wins on the wire (`extra_body` is merged last, which is why `resolveMaxOut` ranks it above `Options`), so a scaled `num_predict` could not take effect. Such a row is detected and the retry **skipped** — plain parse error, one WARN, no inert booking; tune `extra_body.max_tokens` there instead. A negative value is fatal at boot / `422`s the settings write (**V19** — the off-semantics are the whole range ≤ 1, so `0` and `1` are ordinary settings and only a negative factor has no reading at all) | | `CTX_DREAM_DECIDE_TIE_ODDS` | `1.618034` (φ) | hot | Near-tie distance of **decide-mode** link writing, as an odds ratio: of one block's gated links, the most confident (the anchor) is written plus every link whose odds lie within this factor of the anchor's; the rest is not a tie and is dropped before the hard cap (`links_untied` on the `dream: decide links finished` log line). Relative, not a threshold, because the read probability is overconfident while the per-block ranking is reliable (calibration 2026-10-03, see architecture *Decide mode*). **`0` disables the rule** (every gated link up to the hard cap, the pre-rule behaviour); `1` keeps the anchor and exact ties only. A value in (0, 1), a negative one, NaN or +Inf is fatal at boot / `422`s the settings write (**V36**). Read only when `CTX_DREAM_DECIDE_MODE` is `eval` or `all` | | `CTX_DREAM_LANGUAGE` | empty | hot | The **corpus language**: one knob for every surface that produces readable text — the daily synthesis report (the 03:00 iteration and `POST /api/synthesize/daily`), the LLM **cluster labels** (see `_LABEL_*` below), the **root map's fixed scaffolding plus its number formatting** (headings, coverage wording, freeze clauses, footer; `1.190`/`97,9 %` vs `1,190`/`97.9%`), and the `digest.mode=stub` pointer that forwards the old topic map to it. The per-block dream pipeline is unaffected. Empty = legacy behavior: German report, title `Tagesbericht `, tag `tagesbericht`, German map scaffolding. A BCP-47-style tag switches title (`Daily Report `), tag (`daily-report`) and system prompt together; the **primary subtag** decides, so `de-DE` stays German. The map has exactly **two** scaffolding tables — empty or any `de*` tag keeps the frozen German, every other tag renders **English** — while the LLM-produced labels follow the tag itself, so an unusual tag yields English scaffolding around labels in that language rather than a German/foreign mix. ⚠ The title is half the `(category, title, scope)` upsert key — changing this **starts a new report series**, it does not rename the old one (see [api](api.md#endpoints)). Validated (V14): `^[a-z]{2,3}(-[a-z0-9]{2,8})*$`, ≤ 35 chars — a malformed value aborts boot / 422s the settings write, because the tag is interpolated into an LLM system prompt | -| `CTX_DREAM_JSON_MODE` | `strict` | hot | Wire policy of the four dream stages that **parse** their answer — link evaluation, keyword extraction, the recurrence confirm and the temporal Phase-2 review. `strict` is the historical wire and sends the dialect's JSON-mode marker (`response_format: {"type":"json_object"}` on the OpenAI path, top-level `"format":"json"` on the Ollama path); `off` sends plain chat, leaving the JSON contract in the prompt and the pipeline's **local parsers as the only validator** — which is what they already are on every well-formed answer, since nothing downstream trusts the grammar. Empty reads as `strict` (legacy sentinel, same doctrine as `CTX_DREAM_LANGUAGE`); any other value is fatal at boot / `422`s the settings write (**V20** — deliberately not a tolerant WARN-and-reset: an unknown value would silently mean `strict`, so a typo'd opt-out would keep the grammar on the wire while the settings surface renders the operator's value, and the symptom they were chasing is a slow backend, not a wrong one). **Why turn it off:** a grammar-enforcing runtime pays for the constraint in decode throughput (the report on issue #33 measures roughly a halving on an oMLX relay, with speculative decoding disabled while a grammar is active; not reproduced in this repo), and the dream pipeline is the throughput floor of the corpus. The key is hot, so it can be A/B-tested on a live install without a restart. **What it does not cover:** the **daily synthesis** never sends the marker under either setting — its answer is prose stored verbatim as the report body, so a grammar there is not a validator but a corruptor. **Risk when off:** answers that arrive as prose (rather than bare or fenced JSON) become parse errors on the stages listed as `FAIL` below — for evaluation and recurrence that is a transient cooldown and a re-pick, for temporal a silently dropped finding. Watch `metadata.parse_format` on the `context_llm_log` rows for a drift-form shift after flipping it | +| `CTX_DREAM_JSON_MODE` | `strict` | hot | Wire policy of the four dream stages that **parse** their answer — link evaluation, keyword extraction, the recurrence confirm and the temporal Phase-2 review. `strict` is the historical wire and sends the dialect's JSON-mode marker (`response_format: {"type":"json_object"}` on the OpenAI path, top-level `"format":"json"` on the Ollama path; **the `anthropic` path sends no marker under either setting** — structured outputs need a JSON schema ctx does not carry, so a dream backend on that wire behaves as `off` regardless, and setting `off` there only makes the row say what the wire does); `off` sends plain chat, leaving the JSON contract in the prompt and the pipeline's **local parsers as the only validator** — which is what they already are on every well-formed answer, since nothing downstream trusts the grammar. Empty reads as `strict` (legacy sentinel, same doctrine as `CTX_DREAM_LANGUAGE`); any other value is fatal at boot / `422`s the settings write (**V20** — deliberately not a tolerant WARN-and-reset: an unknown value would silently mean `strict`, so a typo'd opt-out would keep the grammar on the wire while the settings surface renders the operator's value, and the symptom they were chasing is a slow backend, not a wrong one). **Why turn it off:** a grammar-enforcing runtime pays for the constraint in decode throughput (the report on issue #33 measures roughly a halving on an oMLX relay, with speculative decoding disabled while a grammar is active; not reproduced in this repo), and the dream pipeline is the throughput floor of the corpus. The key is hot, so it can be A/B-tested on a live install without a restart. **What it does not cover:** the **daily synthesis** never sends the marker under either setting — its answer is prose stored verbatim as the report body, so a grammar there is not a validator but a corruptor. **Risk when off:** answers that arrive as prose (rather than bare or fenced JSON) become parse errors on the stages listed as `FAIL` below — for evaluation and recurrence that is a transient cooldown and a re-pick, for temporal a silently dropped finding. Watch `metadata.parse_format` on the `context_llm_log` rows for a drift-form shift after flipping it | | `CTX_DREAM_BACKOFF_MODE` / `_FACTOR` / `_MIN` / `_GRACE` / `_CAP` / `_INERT_OFFSET` | `exp` / `1.6` / `12h` / `0` / `45d` / `7` | hot | Re-dream back-off by eval count (`exp`/`log`/`linear`/`off`). Cooldown grows from `MIN` (n=0) to `CAP`: fresh blocks re-dream sub-day, mature blocks back off to the cap. `_MIN`/`_CAP` take a duration suffix — `h`/`d`/`w`/`m` (30d)/`y` (365d), e.g. `12h`, `45d`, `1w` (bare number = hours). `_INERT_OFFSET` starts a no-links cycle further up the curve | | `CTX_PROMPT_VERSION` | `v5.2` | hot | Generator-prompt version (`v5.2` default, `v6` opt-in graded confidence) | | `CTX_TIMEZONE` | `Europe/Berlin` | hot | Cyclic-temporal phase calculation | @@ -415,7 +415,7 @@ run journal is unchanged and still carries the class `block_write_failed` and no The backend pool (`context_backends`) is the living LLM configuration — `ctx backends` lists and manages it, the web UI at `/settings/backends` does the same visually. -Two ways in on a fresh installation. Interactively, [`ctx init`](#first-run-ctx-init) asks for host, protocol and the two models, seeds the pool and probes what it wrote — the same code path, wrapped in prompts. Non-interactively (scripts, CI, cloud-init), `ctx backends seed` writes the first two rows without any interaction: +Two ways in on a fresh installation. Interactively, [`ctx init`](#first-run-ctx-init) asks for host, protocol and the two models (for the `anthropic` protocol also the chat model's context window and a separate embedding host, since the Messages API serves no embeddings), seeds the pool and probes what it wrote — the same code path, wrapped in prompts. Non-interactively (scripts, CI, cloud-init), `ctx backends seed` writes the first two rows without any interaction: ```bash ctx backends seed --host http://gpu-host:11434 --model qwen3 --embed-model qwen3-embedding @@ -432,12 +432,22 @@ The SeedSpec file has one object per role; everything except `host` and `model` } ``` +Hosted Claude through the native Messages API (`protocol: "anthropic"`, see [api — Anthropic](api.md#backend-pool)) is a two-host spec: the chat leg needs the key (sealed into a secret, as below) and `num_ctx` (the wire has no window discovery), the embed leg stays on a local `ollama`/`openai` backend because the Messages API serves no embeddings. `base_url` is the API root without `/v1`: + +```json +{ + "chat": {"host": "https://api.anthropic.com", "protocol": "anthropic", "model": "claude-opus-5-5", "num_ctx": 100000, "api_key": "sk-ant-…"}, + "embed": {"host": "http://gpu-host:11434", "model": "qwen3-embedding"} +} +``` + What to know before running it: - **A server-admin key is required.** A tenant-admin's writes would be pinned to its own tenant scope — the seed refuses instead of reporting success over a shared pool that stays empty. - **The rows are seeded `full-trust`**, which is what lets them serve your own blocks (the pool's default block sensitivity is `credentials`, and a `public` backend is filtered out of that chain). Set `"trust"` per role in the spec to deviate deliberately. - **`api_key` never lands in the table.** It is sealed into an F2 secret first and the row carries only the reference, so the server needs a configured `CTX_SECRETS_KEY` (`openssl rand -hex 32`) before a spec with `api_key` can be seeded. Without one the run aborts and writes nothing — there is no plaintext fallback. - **It is idempotent per row.** A run interrupted between the two creates is completed by simply running it again. Rows *outside* the pair it manages abort the run instead (`--force` overrides only that check), so an already-configured pool is never touched by accident. +- **The anthropic leg is validated on the spec.** `protocol: "anthropic"` on the embed leg, without a key, or without `num_ctx` aborts before anything is sealed or written — the same rules the server enforces with a 422 on `backend-create`. - **A seed file is transient, not config.** It carries the api_key in the clear: `chmod 0600`, delete it after the seed, never commit it. - **A pool that exists is not a pool that serves.** `ctx init` probes each serving row and names the dead ones; `ctx backends test ` does it for a single row. Rows named `herbert-chat` / `llama-embed` on `http://localhost:11434` are the residue of an env bootstrap that ran with nothing configured — inside the ctx container that URL is the container itself. They have to *leave* the pool, and `--force` does not do that for you: it only skips the foreign-row guard and then creates the seed's own rows, leaving the legacy pair enabled at priority 100 next to them (dead members of every failover chain, one health-probe timeout each). Delete them first — `ctx backends delete ` — then run `ctx backends seed`. Parking them instead (`ctx backends update '{"enabled":false}'`) keeps them out of the chains but leaves the seed needing `--force`. @@ -447,7 +457,7 @@ To take a GPU host (e.g. `herbert`) down for maintenance without dropping in-fli ## Models & hardware -ctx is **engine-agnostic**: every LLM role speaks either the OpenAI-compatible or the Ollama wire protocol (per-backend `protocol`), so any provider serving one of those works — local llama.cpp / Ollama or a remote API. Models are configuration, not code: each `context_backends` row carries a per-role `model_map` (see [architecture — backend pool](architecture.md#backend-pool-f3-migrations-053055)). The models below are the **tested production reference**, not hard requirements. Beyond the LLM backends, ctx itself is a single Go daemon plus PostgreSQL 18 + pgvector — no GPU needs of its own. +ctx is **engine-agnostic**: every LLM role speaks the OpenAI-compatible or the Ollama wire protocol (per-backend `protocol`), and the chat-class roles (synthesis, translate, dream, digest, chat) can also run on the native Anthropic Messages API (`anthropic`, hosted Claude — no embeddings or rerank there), so any provider serving one of those works — local llama.cpp / Ollama or a remote API. Models are configuration, not code: each `context_backends` row carries a per-role `model_map` (see [architecture — backend pool](architecture.md#backend-pool-f3-migrations-053055)). The models below are the **tested production reference**, not hard requirements. Beyond the LLM backends, ctx itself is a single Go daemon plus PostgreSQL 18 + pgvector — no GPU needs of its own. | Role(s) | Reference model | Host type | Purpose | |---|---|---|---| @@ -713,6 +723,10 @@ docker exec n8n-db-1 psql -U "$CONTEXT_DB_USER" -d "$CONTEXT_DB" -c \ WHERE metadata ? 'supersedes_repair';" ``` +### Migration 155: `anthropic` joins the backend wire protocols + +`context_backends.protocol` was pinned by a `CHECK` to `openai`, `ollama` and `rerank`. Migration 155 widens it to include `anthropic` (the native Messages API wire, [api — Anthropic](api.md#backend-pool)). It resolves the baseline's unnamed constraint from `pg_constraint` by table and definition, drops it and re-adds it as `context_backends_protocol_check`; no row changes, the column keeps its `DEFAULT 'openai'`. Nothing to do at upgrade time. **Downgrading** a daemon below this migration with an `anthropic` row still in the pool makes that row dispatch onto the Ollama wire (the old fallthrough for unknown protocols) — disable or delete such rows before rolling the binary back; the widened `CHECK` itself is harmless to an older binary. + ### `graph_overview.csr_loader`: the rebuild's input substrate `CTX_GRAPH_OVERVIEW_CSR_LOADER` (default `false`, hot) switches how the rebuild gets its graph into memory. It changes no result — the partition, the modularity and the intra-cluster degrees are byte-identical either way, and that identity is a gate, not a hope. diff --git a/go/internal/cli/backends_seed.go b/go/internal/cli/backends_seed.go index 7677a29e..5dc3480d 100644 --- a/go/internal/cli/backends_seed.go +++ b/go/internal/cli/backends_seed.go @@ -281,8 +281,22 @@ func seedRowFor(in seedBackend, defaultName string, roles []string, label string } switch backends.Protocol(protocol) { case backends.ProtocolOllama, backends.ProtocolOpenAI: + case backends.ProtocolAnthropic: + // The Messages API is chat-only (no embeddings endpoint), keyed, and + // has no window discovery — the server's validation says the same + // (422), but a seed is the first contact of a fresh install and + // should fail on the spec, before any secret is sealed. + if label == "embed" { + return seedRow{}, fmt.Errorf("%s: protocol anthropic serves no embeddings — give the embed leg an ollama or openai backend", label) + } + if in.APIKey == "" && in.APIKeyRef == "" { + return seedRow{}, fmt.Errorf("%s: protocol anthropic requires api_key or api_key_ref", label) + } + if in.NumCtx <= 0 { + return seedRow{}, fmt.Errorf("%s: protocol anthropic requires num_ctx (the model's context window, e.g. 100000) — there is no window discovery on this wire", label) + } default: - return seedRow{}, fmt.Errorf("%s: protocol %q is not one of ollama, openai", label, protocol) + return seedRow{}, fmt.Errorf("%s: protocol %q is not one of ollama, openai, anthropic", label, protocol) } trust := strings.TrimSpace(in.Trust) if trust == "" { diff --git a/go/internal/cli/backends_seed_test.go b/go/internal/cli/backends_seed_test.go index 38cd8e0c..f7b033c5 100644 --- a/go/internal/cli/backends_seed_test.go +++ b/go/internal/cli/backends_seed_test.go @@ -146,6 +146,9 @@ func TestSeedRowsRejectBadSpecs(t *testing.T) { {"chat host missing", seedSpec{Chat: seedBackend{Model: "m"}, Embed: ok}, "chat: host is required"}, {"embed model missing", seedSpec{Chat: ok, Embed: seedBackend{Host: "http://x"}}, "embed: model is required"}, {"bad protocol", seedSpec{Chat: seedBackend{Host: "http://x", Model: "m", Protocol: "grpc"}, Embed: ok}, "protocol"}, + {"anthropic on embed", seedSpec{Chat: ok, Embed: seedBackend{Host: "https://api.anthropic.com", Model: "m", Protocol: "anthropic", APIKey: "k", NumCtx: 1}}, "no embeddings"}, + {"anthropic keyless", seedSpec{Chat: seedBackend{Host: "https://api.anthropic.com", Model: "m", Protocol: "anthropic", NumCtx: 100000}, Embed: ok}, "api_key"}, + {"anthropic no num_ctx", seedSpec{Chat: seedBackend{Host: "https://api.anthropic.com", Model: "m", Protocol: "anthropic", APIKey: "k"}, Embed: ok}, "num_ctx"}, {"bad trust", seedSpec{Chat: seedBackend{Host: "http://x", Model: "m", Trust: "sort-of"}, Embed: ok}, "trust"}, {"key and ref", seedSpec{Chat: seedBackend{Host: "http://x", Model: "m", APIKey: "a", APIKeyRef: "b"}, Embed: ok}, "mutually exclusive"}, {"negative num_ctx", seedSpec{Chat: seedBackend{Host: "http://x", Model: "m", NumCtx: -1}, Embed: ok}, "num_ctx"}, @@ -333,3 +336,23 @@ func TestSeedRowLeadRoleIsTheLegsReason(t *testing.T) { type errorString string func (e errorString) Error() string { return string(e) } + +// TestSeedRowsAnthropicChatLeg: the anthropic protocol seeds a keyed, +// windowed chat row next to a local embed row — the mixed-protocol pair a +// hosted-Claude install needs (the Messages API serves no embeddings). +func TestSeedRowsAnthropicChatLeg(t *testing.T) { + rows := mustRows(t, seedSpec{ + Chat: seedBackend{Host: "https://api.anthropic.com", Protocol: "anthropic", Model: "claude-opus-5-5", APIKey: "sk-ant", NumCtx: 100000}, + Embed: seedBackend{Host: "http://gpu:11434", Model: "qwen3-embed"}, + }) + chat := rows[seedChatName] + if chat.payload["protocol"] != "anthropic" || chat.payload["num_ctx"] != 100000 { + t.Errorf("chat payload = %v, want protocol anthropic with num_ctx 100000", chat.payload) + } + if chat.payload["api_key_ref"] != seedChatName+"-key" || chat.secretValue != "sk-ant" { + t.Errorf("chat key must seal into %s-key: payload=%v secret=%q", seedChatName, chat.payload, chat.secretValue) + } + if rows[seedEmbedName].payload["protocol"] != "ollama" { + t.Errorf("embed payload = %v, want the ollama default", rows[seedEmbedName].payload) + } +} diff --git a/go/internal/cli/init.go b/go/internal/cli/init.go index 3d9fdeea..f27e227b 100644 --- a/go/internal/cli/init.go +++ b/go/internal/cli/init.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "strconv" "strings" "time" @@ -392,24 +393,55 @@ func askSeedSpec(ask backendsAsk, missing []string) (seedSpec, error) { if host == "" { return seedSpec{}, errors.New("no host given — nothing was written") } - protocol := ask.line(" ? Protocol, ollama or openai [ollama]: ") + protocol := ask.line(" ? Protocol, ollama, openai or anthropic [ollama]: ") if protocol == "" { protocol = string(backends.ProtocolOllama) } chatModel := ask.line(" ? Chat model: ") embedModel := ask.line(" ? Embedding model: ") - key := "" + chat := seedBackend{Host: host, Protocol: protocol, Model: chatModel} + embed := seedBackend{Host: host, Protocol: protocol, Model: embedModel} + if backends.Protocol(protocol) == backends.ProtocolAnthropic { + // The Messages API is chat-only and keyed: the embed leg needs its + // own host and wire, the chat leg a declared window and the key. + chat.NumCtx = askNumCtx(ask, " ? Context window of the chat model in tokens [100000]: ", 100000) + embed.Host = ask.line(" ? Embedding host (e.g. http://localhost:11434): ") + if embed.Host == "" { + return seedSpec{}, errors.New("no embedding host given — the anthropic wire serves no embeddings, nothing was written") + } + embed.Protocol = ask.line(" ? Embedding protocol, ollama or openai [ollama]: ") + if embed.Protocol == "" { + embed.Protocol = string(backends.ProtocolOllama) + } + v, err := ask.secret(" ? Anthropic API key (not echoed): ") + if err != nil { + return seedSpec{}, err + } + chat.APIKey = v + return seedSpec{Chat: chat, Embed: embed}, nil + } if answerIsYes(ask.line(" ? Does the host need an API key? [y/N]: ")) { v, err := ask.secret(" ? API key (not echoed): ") if err != nil { return seedSpec{}, err } - key = v + chat.APIKey, embed.APIKey = v, v + } + return seedSpec{Chat: chat, Embed: embed}, nil +} + +// askNumCtx reads an integer answer with a default for the empty line; a +// non-numeric answer falls back to the default as well — the server's +// validation is the gate, the wizard only collects. +func askNumCtx(ask backendsAsk, question string, def int) int { + answer := strings.TrimSpace(ask.line(question)) + if answer == "" { + return def + } + if n, err := strconv.Atoi(answer); err == nil && n > 0 { + return n } - return seedSpec{ - Chat: seedBackend{Host: host, Protocol: protocol, Model: chatModel, APIKey: key}, - Embed: seedBackend{Host: host, Protocol: protocol, Model: embedModel, APIKey: key}, - }, nil + return def } func answerIsYes(answer string) bool { diff --git a/go/internal/cli/init_test.go b/go/internal/cli/init_test.go index ff8bb1bb..9871d05d 100644 --- a/go/internal/cli/init_test.go +++ b/go/internal/cli/init_test.go @@ -733,3 +733,56 @@ func containsHelper(s, substr string) bool { } return false } + +// TestInitBackendsStepAnthropicAsksEmbedLegSeparately: choosing the anthropic +// wire turns the single-host wizard into a two-host one — the Messages API +// serves no embeddings — and collects the window and the key the chat leg +// needs without the "does the host need a key?" detour (it always does). +func TestInitBackendsStepAnthropicAsksEmbedLegSeparately(t *testing.T) { + m := newInitMock(t, serverAdmin, nil) + ask, secretCalls := scriptedAsk([]string{ + "https://api.anthropic.com", "anthropic", "claude-opus-5-5", "qwen3-embedding", + "", // context window → default 100000 + "http://gpu:11434", // embedding host + "", // embedding protocol → ollama default + }, "sk-ant-live") + + var ok bool + out := captureStdout(t, func() { ok = runBackendsStep(m.client(), ask) }) + if !ok { + t.Fatalf("anthropic seed must succeed; output:\n%s", out) + } + if *secretCalls != 1 { + t.Errorf("the key prompt must run exactly once (%d calls)", *secretCalls) + } + if strings.Contains(out, "sk-ant-live") { + t.Error("the key was echoed") + } + byName := map[string]map[string]any{} + for _, c := range m.creates { + name, _ := c["name"].(string) + byName[name] = c + } + chat, embed := byName[seedChatName], byName[seedEmbedName] + if chat == nil || embed == nil { + t.Fatalf("wizard did not create both rows: %v", m.creates) + } + if chat["protocol"] != "anthropic" || chat["base_url"] != "https://api.anthropic.com" { + t.Errorf("chat row = %v", chat) + } + if n, _ := chat["num_ctx"].(float64); int(n) != 100000 { + // The mock round-trips the payload through JSON, hence float64. + if n2, ok := chat["num_ctx"].(int); !ok || n2 != 100000 { + t.Errorf("chat num_ctx = %v, want the 100000 default", chat["num_ctx"]) + } + } + if chat["api_key_ref"] != seedChatName+"-key" { + t.Errorf("chat api_key_ref = %v", chat["api_key_ref"]) + } + if embed["protocol"] != "ollama" || embed["base_url"] != "http://gpu:11434" { + t.Errorf("embed row = %v, want the separately asked ollama host", embed) + } + if _, has := embed["api_key_ref"]; has { + t.Errorf("the Anthropic key must not be attached to the embed row: %v", embed) + } +} diff --git a/go/internal/handler/backends_manage.go b/go/internal/handler/backends_manage.go index ee1e5fda..e9f89025 100644 --- a/go/internal/handler/backends_manage.go +++ b/go/internal/handler/backends_manage.go @@ -799,10 +799,17 @@ func dataCollectionEscapeOn(b *backends.Backend) bool { func probeBackendURL(ctx context.Context, b *backends.Backend, checks map[string]string) bool { url := b.Host - switch b.ProviderClass { - case backends.ProviderLlamaCpp: + anthropic := b.Protocol == backends.ProtocolAnthropic + switch { + case anthropic: + // The Messages API root answers nothing useful; /v1/models is the + // cheapest keyed GET (same root convention: base_url + "/v1/..."). + // A bad key is a 401 here, which the generic branch below reports as + // checks.auth — the chat probe then never runs a priced call. + url += "/v1/models" + case b.ProviderClass == backends.ProviderLlamaCpp: url += "/health" - case backends.ProviderOpenRouter: + case b.ProviderClass == backends.ProviderOpenRouter: // /v1/key matches the chat path convention (base_url is the API root // without the version segment, b.Host+"/v1/..."). url += "/v1/key" @@ -814,7 +821,13 @@ func probeBackendURL(ctx context.Context, b *backends.Backend, checks map[string checks["base_url"] = "error: invalid URL" return false } - if b.APIKey != "" { + switch { + case anthropic: + httpReq.Header.Set("anthropic-version", "2023-06-01") + if b.APIKey != "" { + httpReq.Header.Set("x-api-key", b.APIKey) + } + case b.APIKey != "": httpReq.Header.Set("Authorization", "Bearer "+b.APIKey) } resp, err := http.DefaultClient.Do(httpReq) //nolint:gosec // G704: admin-gated runtime data; reaching arbitrary URLs is this action's function diff --git a/go/internal/handler/backends_test_anthropic_test.go b/go/internal/handler/backends_test_anthropic_test.go new file mode 100644 index 00000000..8d871b70 --- /dev/null +++ b/go/internal/handler/backends_test_anthropic_test.go @@ -0,0 +1,68 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/GottZ/ctx/internal/backends" +) + +// TestBackendTestAnthropicProbe: an anthropic-protocol backend-test probes +// GET /v1/models with the Messages API headers (x-api-key + +// anthropic-version, no Bearer) and reports base_url/auth ok on a 2xx — the +// generic branch would GET the bare root with a Bearer header, which the +// API answers 404 (unreachable) or 401. +func TestBackendTestAnthropicProbe(t *testing.T) { + var mu sync.Mutex + var paths []string + var hdr http.Header + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, r.URL.Path) + hdr = r.Header.Clone() + mu.Unlock() + if r.URL.Path == "/v1/models" { + _, _ = w.Write([]byte(`{"data":[{"id":"claude-opus-5-5"}]}`)) + return + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + + bp := backends.NewPool(nil, nil) + bp.SeedSnapshotForTest([]backends.Backend{{ + ID: "0190-ant", Name: "anthropic", Host: srv.URL, + Protocol: backends.ProtocolAnthropic, ProviderClass: backends.ProviderGeneric, + Trust: backends.TrustNoCredentials, Locality: backends.LocalityExternal, + Roles: []string{backends.RoleSynthesis}, NumCtx: 100000, + ModelMap: map[string]backends.ModelSpec{"default": {Model: "claude-opus-5-5"}}, + APIKey: "sk-ant-test", APIKeyRef: "anthropic-chat", Enabled: true, + }}) + + rec := manageReqWithPool(t, adminAR(), bp, map[string]any{ + "action": "backend-test", "id": "0190-ant", + }) + if rec.Code != http.StatusOK { + t.Fatalf("backend-test: status %d (body %s)", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + for _, want := range []string{`"base_url":"ok"`, `"auth":"ok"`} { + if !strings.Contains(body, want) { + t.Errorf("backend-test response lacks %s: %s", want, body) + } + } + mu.Lock() + defer mu.Unlock() + if len(paths) != 1 || paths[0] != "/v1/models" { + t.Errorf("probe paths = %v, want exactly /v1/models", paths) + } + if hdr.Get("x-api-key") != "sk-ant-test" || hdr.Get("anthropic-version") == "" { + t.Errorf("probe headers = %v, want x-api-key + anthropic-version", hdr) + } + if hdr.Get("Authorization") != "" { + t.Errorf("probe sent a Bearer header on the anthropic wire: %q", hdr.Get("Authorization")) + } +} diff --git a/go/web/src/lib/backends.ts b/go/web/src/lib/backends.ts index 1bf2957a..f41a2081 100644 --- a/go/web/src/lib/backends.ts +++ b/go/web/src/lib/backends.ts @@ -64,7 +64,7 @@ export function isTrustElevation(prev: string, next: string): boolean { // Wire enums (verified against backends/validate.go, NOT the design doc which // undercounted locality). locality is also auto-derived from base_url on create // and cross-checked against the host, so '' (auto) is the sane default. -export const PROTOCOLS = ['openai', 'ollama', 'rerank'] as const +export const PROTOCOLS = ['openai', 'ollama', 'anthropic', 'rerank'] as const export const PROVIDER_CLASSES = ['generic', 'llamacpp', 'openrouter'] as const export const LOCALITIES = ['local', 'lan', 'external'] as const From 40116df0a8bb3dfc114d889d7ec4da5ce4089009 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 00:38:59 +0200 Subject: [PATCH 08/16] fix(chat): carry model_map params.effort onto the web-chat stream path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chatOptions builds the stream call's Options by hand (the harness owns NumPredict and the think toggle, so it never runs applyModelParams) and copied only temperature and top_p from the chat role's params. The anthropic wire resolves output_config.effort from Options.Extra["effort"] first — the slot applyModelParams fills on the non-stream path — so a row's model_map.chat.params.effort was honoured by synthesis and dream but silently ignored by web chat, which sent extra_body.effort or the default medium instead. docs/api.md promised the params-over-extra_body precedence for both. Found by the adversarial review; the existing tests injected Options.Extra by hand and never went through chatOptions. chatOptions now lifts params.effort into Options.Extra. Test: TestChatOptionsCarriesModelMapEffort (and the nil slot without the param). Co-Authored-By: Claude Fable 5.1 --- go/internal/chat/chat_options_effort_test.go | 34 ++++++++++++++++++++ go/internal/chat/engine.go | 11 +++++++ 2 files changed, 45 insertions(+) create mode 100644 go/internal/chat/chat_options_effort_test.go diff --git a/go/internal/chat/chat_options_effort_test.go b/go/internal/chat/chat_options_effort_test.go new file mode 100644 index 00000000..a8e79279 --- /dev/null +++ b/go/internal/chat/chat_options_effort_test.go @@ -0,0 +1,34 @@ +package chat + +import ( + "testing" + "time" + + "github.com/GottZ/ctx/internal/backends" +) + +// TestChatOptionsCarriesModelMapEffort pins the review finding that the +// harness bypasses applyModelParams: a chat row's params.effort must still +// reach Options.Extra, the slot the anthropic stream wire reads — otherwise +// docs/api.md's "model_map params.effort beats extra_body" holds on the +// non-stream path only. +func TestChatOptionsCarriesModelMapEffort(t *testing.T) { + e := &Engine{cfg: Config{}.withDefaults(), now: time.Now} + b := backends.Backend{ModelMap: map[string]backends.ModelSpec{ + "chat": {Model: "claude-opus-5-5", Params: map[string]any{"effort": "xhigh", "temperature": 0.2}}, + }} + opts := e.chatOptions(b, 100) + if opts.Extra["effort"] != "xhigh" { + t.Errorf("Options.Extra = %v, want effort xhigh from model_map params", opts.Extra) + } + if opts.Temperature != 0.2 || opts.NumPredict != 100 { + t.Errorf("temperature/num_predict = %v/%d, want 0.2/100", opts.Temperature, opts.NumPredict) + } + + // Without the param the slot stays nil — the openai wire's extra_body + // merge and the anthropic default are untouched. + plain := e.chatOptions(backends.Backend{ModelMap: map[string]backends.ModelSpec{"chat": {Model: "m"}}}, 100) + if plain.Extra != nil { + t.Errorf("Options.Extra = %v, want nil without an effort param", plain.Extra) + } +} diff --git a/go/internal/chat/engine.go b/go/internal/chat/engine.go index 2f2b1c2f..3e37caa7 100644 --- a/go/internal/chat/engine.go +++ b/go/internal/chat/engine.go @@ -607,6 +607,14 @@ func (e *Engine) tokenClamp(b backends.Backend, budgetLeft int) int { return v } +// chatOptions builds the stream call's Options from the chat role's +// model_map params. Deliberately NOT applyModelParams: the harness owns +// NumPredict (tokenClamp) and the think toggle (buildStreamBody's +// reasoning-off doctrine), so only the keys the stream wires read are +// honoured — temperature/top_p for the openai wire, effort for the anthropic +// wire (resolveAnthropicEffort reads Options.Extra["effort"], the same slot +// applyModelParams fills on the non-stream path, so a row's params.effort +// means the same thing on both paths). func (e *Engine) chatOptions(b backends.Backend, numPredict int) llm.Options { opts := llm.Options{Temperature: 0.7, NumPredict: numPredict} if spec := b.ModelFor("chat"); spec.Params != nil { @@ -616,6 +624,9 @@ func (e *Engine) chatOptions(b backends.Backend, numPredict int) llm.Options { if tp, ok := spec.Params["top_p"].(float64); ok { opts.TopP = tp } + if eff, ok := spec.Params["effort"].(string); ok && eff != "" { + opts.Extra = map[string]any{"effort": eff} + } } return opts } From ded90726ab92ed0bd306ae1154d1d606aea21c71 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 00:40:00 +0200 Subject: [PATCH 09/16] fix(llm): anthropic stream replays thinking, repairs orphans, caches the tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from the three-way review of the anthropic wire against the claude-api reference, the tests and the docs. Each one is a request the Messages API would have answered 400 to, or a cache write paid for nothing. Thinking blocks ride along. The Claude 5 models think on every answer and require the FINAL assistant turn of a tool loop to start with its thinking block (empty text under the default display, the signature carries it). The parser now re-assembles thinking (thinking_delta + signature_delta, sealed on content_block_stop) and redacted_thinking blocks into StreamResult.ProviderBlocks; the harness puts them on the assistant ChatMsg it appends for the next iteration (ChatMsg.ProviderBlocks, json:"-" so the OpenAI wire never sees it, not persisted — later turns replay without them, which the API accepts for non-final turns), and anthropicMessages emits them first in the assistant turn. Without this the second iteration of every tool-using turn was a 400. Orphaned tool_use is repaired. A turn interrupted between the model's tool call and the tool's result leaves an assistant tool_use without a tool_result in the persisted history, and the API rejects that on every later turn of the session. anthropicMessages now answers every unanswered tool_use with a synthetic is_error tool_result before the next user text (and at the end of the history). Tool calls without ids — an openai-wire row earlier in the session; llama.cpp streams calls without one — get synthetic ids, matched in order to the tool rows that follow. Prompt cache actually hits. Every pipeline ends its system prompt with promptguard.Rule(nonce), fresh per call; one cached block carrying it would be written on every call and read never (cache_creation > 0, cache_read == 0, a pure 1.25x surcharge). anthropicSystem splits the rule into a second, unmarked block, so the stable prefix is the breakpoint. The stream path additionally sends the top-level automatic cache_control, so the growing history is read from cache across iterations and turns. Smaller: message_delta usage copies each input counter only when it is itself present, so a delta carrying input_tokens alone cannot zero the cache counters; a nameless tool_use block (proxy artefact) is dropped instead of dispatched as an empty call; StreamError exposes its provider error type and backends.Classify maps the anthropic types inside a mid-stream error event onto the classes the equivalent status codes get (overloaded_error → gateway failover, rate_limit_error → rate limit, authentication_error → auth, invalid_request_error → bad request, api_error → stop) — OpenRouter's numeric codes keep the stop default; handleFrame split into per-event methods (cyclop); the package doc names the model floor (effort is always sent) and why raw net/http over the Go SDK. Tests: thinking/redacted blocks carried and placed first (fixture now opens with the thinking block Opus 5.5 always sends), orphan and empty-id repair, per-field usage, nameless tool_use, top-level cache_control on the stream path and its absence on the non-stream path, the promptguard split (byte-identical prefix across nonces), StreamError classification. Co-Authored-By: Claude Fable 5.1 --- go/internal/backends/classify.go | 30 +++ go/internal/chat/engine.go | 5 +- go/internal/llm/anthropic.go | 46 +++- go/internal/llm/anthropic_stream.go | 254 ++++++++++++++---- go/internal/llm/anthropic_stream_test.go | 220 +++++++++++++++ go/internal/llm/anthropic_wire_test.go | 48 ++++ go/internal/llm/stream.go | 21 ++ .../llm/testdata/anthropic_tool_use.sse | 30 ++- 8 files changed, 588 insertions(+), 66 deletions(-) diff --git a/go/internal/backends/classify.go b/go/internal/backends/classify.go index 51dbbdee..995771e3 100644 --- a/go/internal/backends/classify.go +++ b/go/internal/backends/classify.go @@ -100,6 +100,12 @@ func (c ErrClass) Cooldown(retryAfter time.Duration) time.Duration { // Classify maps one attempt error onto the failover catalog. providerClass // refines provider-specific permanent failures (OpenRouter "no providers"). // Context errors must be checked FIRST: a canceled parent wraps everything. +// providerErrorTyper is implemented by llm.StreamError: the provider's error +// type of an error event delivered inside an HTTP-200 stream. +type providerErrorTyper interface { + ProviderErrorType() string +} + func Classify(err error, providerClass string) ErrClass { if err == nil { return ClassOK @@ -135,6 +141,30 @@ func Classify(err error, providerClass string) ErrClass { return ClassBadRequest // unknown 4xx: request-specific, no cooldown } } + // A mid-stream error event (HTTP 200, error object inside the SSE + // stream) carries the provider's error TYPE instead of a status code. + // The anthropic wire names the same types its HTTP errors do, so they map + // onto the classes the equivalent status would get — otherwise a 529 + // overloaded_error would fail over as HTTP but stop the chain as an + // event. Unknown types (OpenRouter's numeric codes) fall through to the + // stop-the-chain default below. + var pe providerErrorTyper + if errors.As(err, &pe) { + switch pe.ProviderErrorType() { + case "overloaded_error": + return ClassGateway + case "api_error": + return ClassServerFault + case "rate_limit_error": + return ClassRateLimit + case "billing_error": + return ClassBilling + case "authentication_error", "permission_error": + return ClassAuth + case "invalid_request_error", "not_found_error", "request_too_large": + return ClassBadRequest + } + } if httpx.IsBackendUnavailable(err) { return ClassTransport } diff --git a/go/internal/chat/engine.go b/go/internal/chat/engine.go index 3e37caa7..2609f86b 100644 --- a/go/internal/chat/engine.go +++ b/go/internal/chat/engine.go @@ -231,7 +231,10 @@ func (e *Engine) RunTurn(ctx context.Context, ar *auth.AuthResult, sess *store.C }); perr != nil { return fmt.Errorf("chat: persist assistant tool_calls: %w", perr) } - msgs = append(msgs, llm.ChatMsg{Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls}) + // ProviderBlocks ride along in memory for the next iteration of THIS + // turn (the anthropic wire needs its thinking block back ahead of the + // tool_use it introduced); they are not persisted. + msgs = append(msgs, llm.ChatMsg{Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls, ProviderBlocks: res.ProviderBlocks}) for _, call := range res.ToolCalls { if serr := sink.Event("tool_call", map[string]any{ diff --git a/go/internal/llm/anthropic.go b/go/internal/llm/anthropic.go index 2e9eb8af..2ec4564b 100644 --- a/go/internal/llm/anthropic.go +++ b/go/internal/llm/anthropic.go @@ -34,7 +34,20 @@ // the llmlog metadata (ApplyProviderTelemetry). // - 529 overloaded_error is a plain non-2xx: httpx.StatusError carries it // into backends.Classify as ClassGateway (failover), 429 as -// ClassRateLimit with the Retry-After cooldown, 401 as ClassAuth. +// ClassRateLimit with the Retry-After cooldown, 401 as ClassAuth. The +// same types inside a mid-stream error event classify the same way +// (StreamError.ProviderErrorType). +// - The wire targets the Claude 5 family (claude-opus-5-5 in production). +// output_config.effort is sent on every call; models that predate it +// (Sonnet 4.5, Haiku 4.5) reject the field, and on Opus 4.7/4.8 an +// omitted thinking field means NO thinking — neither is a supported row. +// +// Raw net/http rather than the official Go SDK, like every other ctx wire: +// the backend pool, the dispatch invariant (one pinned +// http.NewRequestWithContext per wire file, chat/dispatch_invariant_test.go) +// and the failover classifier all key on ctx's own request/response path, +// and the request shape is small enough that an SDK would add a dependency +// without removing any of that. package llm import ( @@ -49,6 +62,7 @@ import ( "github.com/GottZ/ctx/internal/backends" "github.com/GottZ/ctx/internal/httpx" + "github.com/GottZ/ctx/internal/promptguard" ) const ( @@ -108,6 +122,9 @@ type anthropicRequest struct { Stream bool `json:"stream,omitempty"` OutputConfig anthropicOutputConfig `json:"output_config"` Tools []anthropicTool `json:"tools,omitempty"` + // CacheControl at the top level is the API's automatic breakpoint on the + // last cacheable block — set on the multi-turn stream path only. + CacheControl *anthropicCacheControl `json:"cache_control,omitempty"` } // --- response shape ---. @@ -172,13 +189,30 @@ func anthropicMaxTokens(opts Options) int { return anthropicDefaultMaxTokens } +// promptguardRuleMarker is the opening of promptguard.Rule's sentence — the +// per-call nonce rule every pipeline appends to its system prompt. +var promptguardRuleMarker = "Everything between <" + promptguard.GuardTag + " id=" + // anthropicSystem renders the system prompt as the cached top-level block. // An empty prompt yields no system field (the API rejects an empty text // block). +// +// The cache is a byte-exact prefix match, and the pipelines end their +// system prompt with promptguard.Rule(nonce) — a fresh nonce per call. One +// block carrying both would be written to the cache on every call and read +// back never (a pure 1.25x surcharge, visible as cache_creation > 0 with +// cache_read == 0 on every row). So the rule is split off into a second, +// unmarked text block: the stable prefix before it is the breakpoint. func anthropicSystem(systemPrompt string) []anthropicTextBlock { if systemPrompt == "" { return nil } + if i := strings.LastIndex(systemPrompt, promptguardRuleMarker); i > 0 { + return []anthropicTextBlock{ + {Type: "text", Text: systemPrompt[:i], CacheControl: &anthropicCacheControl{Type: "ephemeral"}}, + {Type: "text", Text: systemPrompt[i:]}, + } + } return []anthropicTextBlock{{ Type: "text", Text: systemPrompt, CacheControl: &anthropicCacheControl{Type: "ephemeral"}, @@ -329,10 +363,12 @@ func chatAnthropic(ctx context.Context, b backends.Backend, systemPrompt, userPr CacheReadTokens: cacheRead, CacheCreationTokens: cacheCreate, FinishReason: mapAnthropicStopReason(result.StopReason), - // The message id is the provider's audit handle (support requests - // quote it); the model column stays row-faithful — Anthropic echoes - // the requested id, there is no models-fallback to record — and no - // cost is reported on this wire (billing is read in the console). + // The message id (msg_…) is the audit handle that exists on both the + // stream and the non-stream path (the request-id header is the one + // support quotes; it is not read here). The model column stays + // row-faithful — Anthropic echoes the requested id, there is no + // models-fallback to record — and no cost is reported on this wire + // (billing is read in the console). ProviderRequestID: result.ID, }, nil } diff --git a/go/internal/llm/anthropic_stream.go b/go/internal/llm/anthropic_stream.go index e84a5a74..48411f2e 100644 --- a/go/internal/llm/anthropic_stream.go +++ b/go/internal/llm/anthropic_stream.go @@ -63,14 +63,44 @@ type anthropicToolResultBlock struct { Type string `json:"type"` // "tool_result" ToolUseID string `json:"tool_use_id"` Content string `json:"content"` + IsError bool `json:"is_error,omitempty"` } +// anthropicThinkingBlock / anthropicRedactedThinkingBlock are the reasoning +// blocks the stream parser re-assembles so the harness can replay them +// unchanged (the API requires the final assistant turn of a tool loop to +// start with its thinking block; the text is empty under the default +// display and the signature carries the reasoning). +type anthropicThinkingBlock struct { + Type string `json:"type"` // "thinking" + Thinking string `json:"thinking"` + Signature string `json:"signature"` +} + +type anthropicRedactedThinkingBlock struct { + Type string `json:"type"` // "redacted_thinking" + Data string `json:"data"` +} + +// orphanToolResult is the synthetic answer for a tool_use the history never +// answered (the turn was interrupted between the model's call and the +// tool's result — client gone, context cancelled). The API rejects an +// assistant tool_use that is not followed by its tool_result, and the +// orphan sits in the persisted history for the rest of the session, so +// without this repair every later turn of that session would 400. +const orphanToolResult = "no result recorded: the turn was interrupted before this tool ran" + // anthropicMessages converts the multi-turn history: the joined system text // and the alternating message list. Empty assistant turns without tool // calls are dropped (the API rejects empty content), and an assistant turn // that would end the list is kept — the harness never ends on one (it // appends tool results), and a trailing assistant turn is a prefill the // Claude 5 models reject with a clear 400 rather than something to mask. +// +// Every system row is lifted into the top-level system block — the harness +// puts exactly one, first. A mid-history system row would be moved ahead of +// the conversation (and invalidate the cached prefix); if the harness ever +// injects such rows they belong in messages as role:system turns instead. func anthropicMessages(msgs []ChatMsg) (system string, out []anthropicMessage) { var sys []string appendBlocks := func(role string, blocks ...any) { @@ -81,6 +111,24 @@ func anthropicMessages(msgs []ChatMsg) (system string, out []anthropicMessage) { } out = append(out, anthropicMessage{Role: role, Content: blocks}) } + // Tool-call ids are mandatory on this wire (tool_use.id and the + // matching tool_result.tool_use_id). A history served earlier by an + // openai-wire row can carry EMPTY ids — llama.cpp streams calls without + // one and the openai parser tolerates it — so empty ids are repaired + // here: each nameless call gets a synthetic id, and the tool rows that + // follow with an empty ToolCallID consume those ids in order (the + // harness appends one tool row per call, in call order). open holds the + // ids of the last assistant turn's calls that no tool row has answered + // yet; whatever is still open when the next turn starts (or the history + // ends) gets the synthetic error result. + var pendingIDs, open []string + synthetic := 0 + flushOrphans := func() { + for _, id := range open { + appendBlocks("user", anthropicToolResultBlock{Type: "tool_result", ToolUseID: id, Content: orphanToolResult, IsError: true}) + } + open = open[:0] + } for _, m := range msgs { switch m.Role { case "system": @@ -88,15 +136,30 @@ func anthropicMessages(msgs []ChatMsg) (system string, out []anthropicMessage) { sys = append(sys, m.Content) } case "user": + flushOrphans() appendBlocks("user", anthropicTextBlock{Type: "text", Text: m.Content}) case "assistant": + flushOrphans() var blocks []any + // Reasoning blocks first, verbatim — the API's ordering rule for + // the final assistant turn of a tool loop. + for _, raw := range m.ProviderBlocks { + blocks = append(blocks, raw) + } if m.Content != "" { blocks = append(blocks, anthropicTextBlock{Type: "text", Text: m.Content}) } + pendingIDs = pendingIDs[:0] for _, tc := range m.ToolCalls { + id := tc.ID + if id == "" { + synthetic++ + id = fmt.Sprintf("ctxcall_%d", synthetic) + pendingIDs = append(pendingIDs, id) + } + open = append(open, id) blocks = append(blocks, anthropicToolUseBlock{ - Type: "tool_use", ID: tc.ID, Name: tc.Function.Name, + Type: "tool_use", ID: id, Name: tc.Function.Name, Input: anthropicToolInput(tc.Function.Arguments), }) } @@ -105,9 +168,20 @@ func anthropicMessages(msgs []ChatMsg) (system string, out []anthropicMessage) { } appendBlocks("assistant", blocks...) case "tool": - appendBlocks("user", anthropicToolResultBlock{Type: "tool_result", ToolUseID: m.ToolCallID, Content: m.Content}) + id := m.ToolCallID + if id == "" && len(pendingIDs) > 0 { + id, pendingIDs = pendingIDs[0], pendingIDs[1:] + } + for i, o := range open { + if o == id { + open = append(open[:i], open[i+1:]...) + break + } + } + appendBlocks("user", anthropicToolResultBlock{Type: "tool_result", ToolUseID: id, Content: m.Content}) } } + flushOrphans() return strings.Join(sys, "\n\n"), out } @@ -156,6 +230,12 @@ func chatStreamAnthropic(ctx context.Context, b backends.Backend, model string, Stream: true, OutputConfig: anthropicOutputConfig{Effort: resolveAnthropicEffort(opts, &b)}, Tools: anthropicTools(tools), + // Top-level automatic caching: the API places a breakpoint on the + // last cacheable block, so each iteration of the tool loop (and + // each turn of the session) reads the growing history from cache + // instead of paying full input price for it; the explicit system + // breakpoint stays the stable-prefix anchor. + CacheControl: &anthropicCacheControl{Type: "ephemeral"}, } body, err := buildAnthropicBody(req, opts, &b) if err != nil { @@ -201,11 +281,14 @@ type anthropicStreamFrame struct { Type string `json:"type"` ID string `json:"id"` Name string `json:"name"` + Data string `json:"data"` // redacted_thinking payload } `json:"content_block"` Delta *struct { Type string `json:"type"` Text string `json:"text"` PartialJSON string `json:"partial_json"` + Thinking string `json:"thinking"` + Signature string `json:"signature"` StopReason string `json:"stop_reason"` } `json:"delta"` Usage *anthropicUsage `json:"usage"` @@ -215,18 +298,25 @@ type anthropicStreamFrame struct { } `json:"error"` } +// pendingThinking accumulates one thinking block across its deltas. +type pendingThinking struct { + text strings.Builder + signature string +} + // anthropicStreamState carries the assembly across frames. type anthropicStreamState struct { - onEvent func(StreamEvent) error - content strings.Builder - calls []*pendingToolCall - byIndex map[int]*pendingToolCall - finish string - usage anthropicUsage - stopSeen bool - done bool - messageID string - outputSeen bool + onEvent func(StreamEvent) error + content strings.Builder + calls []*pendingToolCall + byIndex map[int]*pendingToolCall + thinking map[int]*pendingThinking + blocks []json.RawMessage // reasoning blocks, stream order, replayed verbatim + finish string + usage anthropicUsage + stopSeen bool + done bool + messageID string } func (st *anthropicStreamState) emit(ev StreamEvent) error { @@ -244,7 +334,7 @@ func (st *anthropicStreamState) emit(ev StreamEvent) error { // complete turn; EOF before it is a hard error — a mid-stream connection // death must not look like a finished answer. func parseAnthropicSSE(r io.Reader, onEvent func(StreamEvent) error) (*StreamResult, error) { - st := &anthropicStreamState{onEvent: onEvent, byIndex: map[int]*pendingToolCall{}} + st := &anthropicStreamState{onEvent: onEvent, byIndex: map[int]*pendingToolCall{}, thinking: map[int]*pendingThinking{}} br := bufio.NewReader(r) var data []string for { @@ -315,50 +405,111 @@ func (st *anthropicStreamState) handleFrame(payload string) error { st.usage = f.Message.Usage } case "content_block_start": - if f.ContentBlock != nil && f.ContentBlock.Type == "tool_use" { - pc := &pendingToolCall{index: f.Index, id: f.ContentBlock.ID, typ: "function", name: f.ContentBlock.Name} - st.byIndex[f.Index] = pc - st.calls = append(st.calls, pc) - if pc.name != "" { - pc.nameEmitted = true - return st.emit(StreamEvent{ToolCallName: pc.name}) - } - } + return st.onBlockStart(&f) case "content_block_delta": - if f.Delta == nil { + return st.onBlockDelta(&f) + case "content_block_stop": + st.onBlockStop(f.Index) + case "message_delta": + return st.onMessageDelta(&f) + case "message_stop": + st.done = true + } + // ping and unknown frame types: ignored. + return nil +} + +// onBlockStart opens a content block. The API puts a tool_use block's id +// and name on the start frame (never on a later delta, unlike the OpenAI +// index-keyed assembly). A nameless tool_use block — a proxy artefact, not +// an API shape — is dropped rather than persisted as an empty call the +// harness would dispatch and replay as tool_use{id:""} (a 400 on the next +// turn); its input_json_delta frames then fall on no pending call. +func (st *anthropicStreamState) onBlockStart(f *anthropicStreamFrame) error { + if f.ContentBlock == nil { + return nil + } + switch f.ContentBlock.Type { + case "tool_use": + if f.ContentBlock.Name == "" { return nil } - switch f.Delta.Type { - case "text_delta": - if f.Delta.Text != "" { - st.content.WriteString(f.Delta.Text) - return st.emit(StreamEvent{ContentDelta: f.Delta.Text}) - } - case "input_json_delta": - if pc := st.byIndex[f.Index]; pc != nil { - pc.args.WriteString(f.Delta.PartialJSON) - } + pc := &pendingToolCall{index: f.Index, id: f.ContentBlock.ID, typ: "function", name: f.ContentBlock.Name, nameEmitted: true} + st.byIndex[f.Index] = pc + st.calls = append(st.calls, pc) + return st.emit(StreamEvent{ToolCallName: pc.name}) + case "thinking": + st.thinking[f.Index] = &pendingThinking{} + case "redacted_thinking": + // Arrives complete on the block start; replayed as-is. + raw, _ := json.Marshal(anthropicRedactedThinkingBlock{Type: "redacted_thinking", Data: f.ContentBlock.Data}) + st.blocks = append(st.blocks, raw) + } + return nil +} + +// onBlockDelta routes a delta by its own type: text to the sink, tool +// input to the pending call, reasoning to the pending thinking block +// (accumulated for the replay only — never surfaced; empty under the +// default display anyway). +func (st *anthropicStreamState) onBlockDelta(f *anthropicStreamFrame) error { + if f.Delta == nil { + return nil + } + switch f.Delta.Type { + case "text_delta": + if f.Delta.Text != "" { + st.content.WriteString(f.Delta.Text) + return st.emit(StreamEvent{ContentDelta: f.Delta.Text}) } - // thinking_delta / signature_delta: never surfaced. - case "message_delta": - if f.Usage != nil { - st.outputSeen = true - st.usage.OutputTokens = f.Usage.OutputTokens - // Newer servers repeat the (cumulative) input counters here; - // take them when present so the total is the final one. - if f.Usage.InputTokens > 0 || f.Usage.CacheReadInputTokens > 0 || f.Usage.CacheCreationInputTokens > 0 { - st.usage.InputTokens = f.Usage.InputTokens - st.usage.CacheReadInputTokens = f.Usage.CacheReadInputTokens - st.usage.CacheCreationInputTokens = f.Usage.CacheCreationInputTokens - } + case "input_json_delta": + if pc := st.byIndex[f.Index]; pc != nil { + pc.args.WriteString(f.Delta.PartialJSON) } - if f.Delta != nil && f.Delta.StopReason != "" && st.finish == "" { - st.stopSeen = true - st.finish = mapAnthropicStopReason(f.Delta.StopReason) - return st.emit(StreamEvent{FinishReason: st.finish}) + case "thinking_delta": + if pt := st.thinking[f.Index]; pt != nil { + pt.text.WriteString(f.Delta.Thinking) } - case "message_stop": - st.done = true + case "signature_delta": + if pt := st.thinking[f.Index]; pt != nil { + pt.signature = f.Delta.Signature + } + } + return nil +} + +// onBlockStop seals a thinking block into its replay form. +func (st *anthropicStreamState) onBlockStop(index int) { + pt := st.thinking[index] + if pt == nil { + return + } + raw, _ := json.Marshal(anthropicThinkingBlock{Type: "thinking", Thinking: pt.text.String(), Signature: pt.signature}) + st.blocks = append(st.blocks, raw) + delete(st.thinking, index) +} + +// onMessageDelta takes the stop reason and the output count. Newer servers +// repeat the (cumulative) input counters here; each one is taken only when +// it is itself present (> 0), so a delta that carries input_tokens alone +// cannot zero the cache counters message_start reported. +func (st *anthropicStreamState) onMessageDelta(f *anthropicStreamFrame) error { + if f.Usage != nil { + st.usage.OutputTokens = f.Usage.OutputTokens + if f.Usage.InputTokens > 0 { + st.usage.InputTokens = f.Usage.InputTokens + } + if f.Usage.CacheReadInputTokens > 0 { + st.usage.CacheReadInputTokens = f.Usage.CacheReadInputTokens + } + if f.Usage.CacheCreationInputTokens > 0 { + st.usage.CacheCreationInputTokens = f.Usage.CacheCreationInputTokens + } + } + if f.Delta != nil && f.Delta.StopReason != "" && st.finish == "" { + st.stopSeen = true + st.finish = mapAnthropicStopReason(f.Delta.StopReason) + return st.emit(StreamEvent{FinishReason: st.finish}) } return nil } @@ -373,6 +524,7 @@ func (st *anthropicStreamState) result() *StreamResult { CacheReadTokens: cacheRead, CacheCreationTokens: cacheCreate, ProviderRequestID: st.messageID, + ProviderBlocks: st.blocks, } for _, pc := range st.calls { args := bytes.TrimSpace(pc.args.Bytes()) diff --git a/go/internal/llm/anthropic_stream_test.go b/go/internal/llm/anthropic_stream_test.go index 738537aa..8a308d03 100644 --- a/go/internal/llm/anthropic_stream_test.go +++ b/go/internal/llm/anthropic_stream_test.go @@ -11,6 +11,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/http/httptest" "strings" @@ -413,3 +414,222 @@ func TestAnthropicMessagesEdgeCases(t *testing.T) { t.Errorf("folded user turn = %v, want three text blocks", msgs[0].Content) } } + +// --- review follow-ups (adversarial pass, 2026-10-11) ---. + +// TestAnthropicStreamMessageDeltaUsageKeepsCacheCounters: a message_delta +// that repeats input_tokens but omits the cache fields (older servers, +// proxies) must not zero the counters message_start reported. +func TestAnthropicStreamMessageDeltaUsageKeepsCacheCounters(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":10,\"cache_read_input_tokens\":400,\"cache_creation_input_tokens\":50}}}\n\n" + + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"input_tokens\":12,\"output_tokens\":4}}\n\n" + + "data: {\"type\":\"message_stop\"}\n\n")) + })) + defer srv.Close() + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + if err != nil { + t.Fatal(err) + } + if res.CacheReadTokens != 400 || res.CacheCreationTokens != 50 || res.PromptTokens != 12+400+50 || res.CompletionTokens != 4 { + t.Errorf("usage = prompt %d cache %d/%d out %d, want 462 / 400/50 / 4", res.PromptTokens, res.CacheReadTokens, res.CacheCreationTokens, res.CompletionTokens) + } +} + +// TestAnthropicStreamNamelessToolUseDropped: a tool_use block start without +// a name (proxy artefact) yields no call and no event; its input deltas +// fall on no pending call. +func TestAnthropicStreamNamelessToolUseDropped(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":1}}}\n\n" + + "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"\",\"name\":\"\",\"input\":{}}}\n\n" + + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{}\"}}\n\n" + + "data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_ok\",\"name\":\"get_time\",\"input\":{}}}\n\n" + + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\"},\"usage\":{\"output_tokens\":2}}\n\n" + + "data: {\"type\":\"message_stop\"}\n\n")) + })) + defer srv.Close() + var events []StreamEvent + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, collectEvents(&events)) + if err != nil { + t.Fatal(err) + } + if len(res.ToolCalls) != 1 || res.ToolCalls[0].ID != "toolu_ok" { + t.Errorf("tool calls = %+v, want only the named one", res.ToolCalls) + } + names := 0 + for _, ev := range events { + if ev.ToolCallName != "" { + names++ + } + } + if names != 1 { + t.Errorf("tool name events = %d, want 1", names) + } +} + +// TestAnthropicMessagesRepairsEmptyToolIDs: a history served earlier by an +// openai-wire row (llama.cpp streams calls without ids) replays with +// synthetic, matching ids instead of a 400 on tool_use.id. +func TestAnthropicMessagesRepairsEmptyToolIDs(t *testing.T) { + _, msgs := anthropicMessages([]ChatMsg{ + {Role: "user", Content: "q"}, + {Role: "assistant", ToolCalls: []ToolCall{ + {Function: ToolCallFunction{Name: "a", Arguments: json.RawMessage(`{}`)}}, + {ID: "real", Function: ToolCallFunction{Name: "b", Arguments: json.RawMessage(`{}`)}}, + {Function: ToolCallFunction{Name: "c", Arguments: json.RawMessage(`{}`)}}, + }}, + {Role: "tool", Content: "ra"}, + {Role: "tool", Content: "rb", ToolCallID: "real"}, + {Role: "tool", Content: "rc"}, + }) + raw, _ := json.Marshal(msgs) + s := string(raw) + for _, want := range []string{ + `"type":"tool_use","id":"ctxcall_1","name":"a"`, `"type":"tool_use","id":"real","name":"b"`, `"type":"tool_use","id":"ctxcall_2","name":"c"`, + `"tool_use_id":"ctxcall_1","content":"ra"`, `"tool_use_id":"real","content":"rb"`, `"tool_use_id":"ctxcall_2","content":"rc"`, + } { + if !strings.Contains(s, want) { + t.Errorf("messages lack %s:\n%s", want, s) + } + } + if strings.Contains(s, `"id":""`) || strings.Contains(s, `"tool_use_id":""`) { + t.Errorf("an empty id reached the wire:\n%s", s) + } +} + +// TestAnthropicStreamThinkingBlocksReplayFirst: the fixture's thinking block +// (empty text, signature — the default display) is re-assembled into +// StreamResult.ProviderBlocks and placed FIRST in the replayed assistant +// turn, ahead of text and tool_use — the API's ordering rule for the final +// assistant turn of a tool loop on the Claude 5 models. +func TestAnthropicStreamThinkingBlocksReplayFirst(t *testing.T) { + srv := serveFixture(t, "anthropic_tool_use.sse") + defer srv.Close() + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "weather?"}}, nil, Options{NumPredict: 64}, 5*time.Second, nil) + if err != nil { + t.Fatal(err) + } + if len(res.ProviderBlocks) != 1 { + t.Fatalf("ProviderBlocks = %d, want the one thinking block", len(res.ProviderBlocks)) + } + if got := string(res.ProviderBlocks[0]); got != `{"type":"thinking","thinking":"","signature":"sigABC"}` { + t.Errorf("thinking block = %s", got) + } + if res.Content != "Let me check." || len(res.ToolCalls) != 2 { + t.Errorf("content/calls disturbed: %q / %d", res.Content, len(res.ToolCalls)) + } + + _, msgs := anthropicMessages([]ChatMsg{ + {Role: "user", Content: "weather?"}, + {Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls, ProviderBlocks: res.ProviderBlocks}, + {Role: "tool", Content: "21C", ToolCallID: "toolu_01weather"}, + {Role: "tool", Content: "12:00", ToolCallID: "toolu_02time"}, + }) + blocks, _ := msgs[1].Content.([]any) + if len(blocks) != 4 { + t.Fatalf("assistant turn = %d blocks, want thinking + text + 2 tool_use", len(blocks)) + } + first, _ := json.Marshal(blocks[0]) + if !strings.HasPrefix(string(first), `{"type":"thinking"`) { + t.Errorf("first block = %s, want the thinking block", first) + } + + // The openai wire never sees the field. + raw, _ := json.Marshal(ChatMsg{Role: "assistant", Content: "x", ProviderBlocks: res.ProviderBlocks}) + if strings.Contains(string(raw), "thinking") { + t.Errorf("ProviderBlocks leaked into the OpenAI message object: %s", raw) + } +} + +// TestAnthropicStreamRedactedThinkingCarried: a redacted_thinking block +// arrives complete on its start frame and is replayed as-is. +func TestAnthropicStreamRedactedThinkingCarried(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":1}}}\n\n" + + "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"redacted_thinking\",\"data\":\"EmwKAhgB\"}}\n\n" + + "data: {\"type\":\"content_block_stop\",\"index\":0}\n\n" + + "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"ok\"}}\n\n" + + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":1}}\n\n" + + "data: {\"type\":\"message_stop\"}\n\n")) + })) + defer srv.Close() + res, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil) + if err != nil { + t.Fatal(err) + } + if len(res.ProviderBlocks) != 1 || string(res.ProviderBlocks[0]) != `{"type":"redacted_thinking","data":"EmwKAhgB"}` { + t.Errorf("ProviderBlocks = %v", res.ProviderBlocks) + } +} + +// TestAnthropicMessagesRepairsOrphanToolUse: a tool_use the history never +// answered (interrupted turn) gets a synthetic is_error tool_result before +// the next user text, and at the end of the history — otherwise every later +// turn of the session would 400 on this wire. +func TestAnthropicMessagesRepairsOrphanToolUse(t *testing.T) { + _, msgs := anthropicMessages([]ChatMsg{ + {Role: "user", Content: "q1"}, + {Role: "assistant", ToolCalls: []ToolCall{ + {ID: "A", Function: ToolCallFunction{Name: "a", Arguments: json.RawMessage(`{}`)}}, + {ID: "B", Function: ToolCallFunction{Name: "b", Arguments: json.RawMessage(`{}`)}}, + }}, + {Role: "tool", Content: "ra", ToolCallID: "A"}, + {Role: "user", Content: "q2"}, + {Role: "assistant", ToolCalls: []ToolCall{{ID: "C", Function: ToolCallFunction{Name: "c", Arguments: json.RawMessage(`{}`)}}}}, + }) + if len(msgs) != 5 { + t.Fatalf("turns = %d, want user / assistant / user[ra, B-orphan, q2] / assistant / user[C-orphan]", len(msgs)) + } + raw, _ := json.Marshal(msgs[2].Content) + s := string(raw) + if !strings.Contains(s, `"tool_use_id":"A","content":"ra"`) || + !strings.Contains(s, `"tool_use_id":"B","content":"`+orphanToolResult+`","is_error":true`) || + !strings.HasSuffix(s, `{"type":"text","text":"q2"}]`) { + t.Errorf("third turn = %s", s) + } + last, _ := json.Marshal(msgs[4].Content) + if msgs[4].Role != "user" || !strings.Contains(string(last), `"tool_use_id":"C"`) || !strings.Contains(string(last), `"is_error":true`) { + t.Errorf("trailing orphan not repaired: %s %s", msgs[4].Role, last) + } + // An answered call gets no synthetic result. + if strings.Count(s, `"tool_use_id":"A"`) != 1 { + t.Errorf("answered call duplicated: %s", s) + } +} + +// TestAnthropicStreamTopLevelCacheControl: the multi-turn stream path asks +// for the automatic breakpoint on the conversation tail. +func TestAnthropicStreamTopLevelCacheControl(t *testing.T) { + srv, body, _ := captureRequestBody(t, "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\ndata: {\"type\":\"message_stop\"}\n\n") + defer srv.Close() + if _, err := ChatStreamBackend(context.Background(), anthropicStreamBackend(srv.URL), "m", + []ChatMsg{{Role: "user", Content: "hi"}}, nil, Options{NumPredict: 8}, 5*time.Second, nil); err != nil { + t.Fatal(err) + } + var req map[string]any + _ = json.Unmarshal(*body, &req) + if cc, _ := req["cache_control"].(map[string]any); cc["type"] != "ephemeral" { + t.Errorf("top-level cache_control = %v, want {type: ephemeral}", req["cache_control"]) + } +} + +// TestStreamErrorClassifiesLikeItsStatus: the anthropic error types inside an +// HTTP-200 stream map onto the classes the equivalent status codes get. +func TestStreamErrorClassifiesLikeItsStatus(t *testing.T) { + cases := map[string]backends.ErrClass{ + "overloaded_error": backends.ClassGateway, "rate_limit_error": backends.ClassRateLimit, + "authentication_error": backends.ClassAuth, "invalid_request_error": backends.ClassBadRequest, + "api_error": backends.ClassServerFault, "502": backends.ClassServerFault, "": backends.ClassServerFault, + } + for code, want := range cases { + err := fmt.Errorf("wrapped: %w", &StreamError{Code: code, Message: "m"}) + if got := backends.Classify(err, backends.ProviderGeneric); got != want { + t.Errorf("Classify(StreamError %q) = %v, want %v", code, got, want) + } + } +} diff --git a/go/internal/llm/anthropic_wire_test.go b/go/internal/llm/anthropic_wire_test.go index 942dd3f1..088b1810 100644 --- a/go/internal/llm/anthropic_wire_test.go +++ b/go/internal/llm/anthropic_wire_test.go @@ -20,6 +20,7 @@ import ( "github.com/GottZ/ctx/internal/backends" "github.com/GottZ/ctx/internal/httpx" "github.com/GottZ/ctx/internal/llmlog" + "github.com/GottZ/ctx/internal/promptguard" ) // anthropicRecorder captures path, decoded body and headers of one @@ -383,3 +384,50 @@ func TestAnthropicErrorClassification(t *testing.T) { }) } } + +// TestAnthropicSystemSplitsPromptguardRule: the per-call nonce rule the +// pipelines append to the system prompt goes into a second, unmarked block +// so the stable prefix before it can actually hit the cache (one block +// with the nonce inside would be written on every call and read never). +func TestAnthropicSystemSplitsPromptguardRule(t *testing.T) { + rule := promptguard.Rule("0123456789abcdef") + blocks := anthropicSystem("You are ctx. Answer from the sources. " + rule) + if len(blocks) != 2 { + t.Fatalf("blocks = %d, want stable prefix + rule", len(blocks)) + } + if blocks[0].Text != "You are ctx. Answer from the sources. " || blocks[0].CacheControl == nil { + t.Errorf("prefix block = %+v, want the text before the rule, cached", blocks[0]) + } + if blocks[1].Text != rule || blocks[1].CacheControl != nil { + t.Errorf("rule block = %+v, want the rule, uncached", blocks[1]) + } + + // A second call with a fresh nonce keeps the cached block byte-identical. + again := anthropicSystem("You are ctx. Answer from the sources. " + promptguard.Rule("fedcba9876543210")) + if again[0].Text != blocks[0].Text { + t.Errorf("cached prefix differs between nonces: %q vs %q", again[0].Text, blocks[0].Text) + } + + // Without a rule: one cached block, as before. A rule at position 0 + // (nothing stable before it) stays one block too. + if one := anthropicSystem("plain"); len(one) != 1 || one[0].CacheControl == nil { + t.Errorf("plain prompt = %+v", one) + } + if only := anthropicSystem(rule); len(only) != 1 { + t.Errorf("rule-only prompt = %d blocks, want 1", len(only)) + } +} + +// TestAnthropicNonStreamHasNoTopLevelCacheControl: the automatic tail +// breakpoint is a multi-turn (stream) affair; the single-turn pipelines +// keep the explicit system breakpoint only. +func TestAnthropicNonStreamHasNoTopLevelCacheControl(t *testing.T) { + rec := newAnthropicRecorder(t, 200, anthropicRespOK, nil) + if _, err := Chat(context.Background(), anthropicTestBackend(rec.srv.URL), "s", "u", Options{NumPredict: 5}, 5*time.Second); err != nil { + t.Fatal(err) + } + _, body, _ := rec.recorded() + if _, has := body["cache_control"]; has { + t.Errorf("top-level cache_control on the non-stream path: %v", body["cache_control"]) + } +} diff --git a/go/internal/llm/stream.go b/go/internal/llm/stream.go index c73d8450..500996fc 100644 --- a/go/internal/llm/stream.go +++ b/go/internal/llm/stream.go @@ -164,6 +164,15 @@ type ChatMsg struct { Content string `json:"content"` ToolCalls []ToolCall `json:"tool_calls,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"` + // ProviderBlocks are opaque content blocks the provider wants back + // unchanged ahead of the turn's text — the anthropic wire's thinking / + // redacted_thinking blocks, which the API requires on the final + // assistant turn of a tool loop. Carried IN MEMORY across the + // iterations of one turn only (the harness does not persist them; a + // later turn's history replays without them, which the API accepts for + // non-final assistant turns). json:"-": never a field of the OpenAI + // wire's message object. + ProviderBlocks []json.RawMessage `json:"-"` } // StreamEvent is handed to the ChatStream callback as the stream progresses. @@ -193,6 +202,10 @@ type StreamResult struct { CacheReadTokens int CacheCreationTokens int ProviderRequestID string + // ProviderBlocks are the reasoning blocks of the answer (anthropic wire), + // to be echoed back via ChatMsg.ProviderBlocks when the harness replays + // this assistant turn inside the same tool loop. nil on the openai wire. + ProviderBlocks []json.RawMessage } // StreamError is an error event delivered inside an HTTP-200 SSE stream. @@ -204,6 +217,14 @@ type StreamError struct { Message string } +// ProviderErrorType exposes the provider's error type string to the failover +// classifier (backends.Classify) without a backends→llm import: the +// anthropic wire's mid-stream error event names the same types its HTTP +// errors do (overloaded_error, rate_limit_error, …), and the classifier maps +// them onto the classes the equivalent status codes get. OpenRouter's +// numeric codes match nothing there and keep their stop-the-chain default. +func (e *StreamError) ProviderErrorType() string { return e.Code } + func (e *StreamError) Error() string { if e.Code != "" { return fmt.Sprintf("llm: stream error event: %s (code %s)", e.Message, e.Code) diff --git a/go/internal/llm/testdata/anthropic_tool_use.sse b/go/internal/llm/testdata/anthropic_tool_use.sse index aafc6b8e..82a89322 100644 --- a/go/internal/llm/testdata/anthropic_tool_use.sse +++ b/go/internal/llm/testdata/anthropic_tool_use.sse @@ -2,34 +2,46 @@ event: message_start data: {"type":"message_start","message":{"id":"msg_01tool","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":40,"output_tokens":2,"cache_read_input_tokens":0,"cache_creation_input_tokens":512}}} event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Let me check."}} +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sigABC"}} event: content_block_stop data: {"type":"content_block_stop","index":0} event: content_block_start -data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01weather","name":"get_weather","input":{}}} +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Let me check."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01weather","name":"get_weather","input":{}}} event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}} +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":""}} event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"city\": \"Ber"}} +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"city\": \"Ber"}} event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"lin\", \"unit\": \"c\"}"}} +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"lin\", \"unit\": \"c\"}"}} event: content_block_stop -data: {"type":"content_block_stop","index":1} +data: {"type":"content_block_stop","index":2} event: content_block_start -data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_02time","name":"get_time","input":{}}} +data: {"type":"content_block_start","index":3,"content_block":{"type":"tool_use","id":"toolu_02time","name":"get_time","input":{}}} event: content_block_stop -data: {"type":"content_block_stop","index":2} +data: {"type":"content_block_stop","index":3} event: message_delta data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":31}} From f6f4194159ab57f1890f021dc9efc1ae5b388a0b Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 00:40:02 +0200 Subject: [PATCH 10/16] fix(backends): wizard rejects a bad context window and asks for the embed key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ctx init, anthropic branch: a non-numeric context window ("100k") no longer falls back to 100000 silently — the step aborts with the reason before any secret is sealed or any row written (a typo'd window would have seeded a tenth of the prompt budget the operator meant). The separately asked embedding host gets the same "does it need an API key?" question the single-host flow has, so a keyed openai embedding service can be seeded next to the Anthropic chat row. probeBackendURL: the comment claimed the chat probe never runs after a 401 on /v1/models; it does (the row counts as reachable) and 401s the same way, billing nothing — the comment now says so. Tests: TestInitBackendsStepAnthropicRejectsBadWindow; the wizard flow test answers the new embed-key question. Co-Authored-By: Claude Fable 5.1 --- go/internal/cli/init.go | 31 ++++++++++++++++++-------- go/internal/cli/init_test.go | 22 ++++++++++++++++++ go/internal/handler/backends_manage.go | 5 +++-- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/go/internal/cli/init.go b/go/internal/cli/init.go index f27e227b..cfe56229 100644 --- a/go/internal/cli/init.go +++ b/go/internal/cli/init.go @@ -404,7 +404,11 @@ func askSeedSpec(ask backendsAsk, missing []string) (seedSpec, error) { if backends.Protocol(protocol) == backends.ProtocolAnthropic { // The Messages API is chat-only and keyed: the embed leg needs its // own host and wire, the chat leg a declared window and the key. - chat.NumCtx = askNumCtx(ask, " ? Context window of the chat model in tokens [100000]: ", 100000) + numCtx, err := askNumCtx(ask, " ? Context window of the chat model in tokens [100000]: ", 100000) + if err != nil { + return seedSpec{}, err + } + chat.NumCtx = numCtx embed.Host = ask.line(" ? Embedding host (e.g. http://localhost:11434): ") if embed.Host == "" { return seedSpec{}, errors.New("no embedding host given — the anthropic wire serves no embeddings, nothing was written") @@ -413,6 +417,13 @@ func askSeedSpec(ask backendsAsk, missing []string) (seedSpec, error) { if embed.Protocol == "" { embed.Protocol = string(backends.ProtocolOllama) } + if answerIsYes(ask.line(" ? Does the embedding host need an API key? [y/N]: ")) { + v, err := ask.secret(" ? Embedding API key (not echoed): ") + if err != nil { + return seedSpec{}, err + } + embed.APIKey = v + } v, err := ask.secret(" ? Anthropic API key (not echoed): ") if err != nil { return seedSpec{}, err @@ -430,18 +441,20 @@ func askSeedSpec(ask backendsAsk, missing []string) (seedSpec, error) { return seedSpec{Chat: chat, Embed: embed}, nil } -// askNumCtx reads an integer answer with a default for the empty line; a -// non-numeric answer falls back to the default as well — the server's -// validation is the gate, the wizard only collects. -func askNumCtx(ask backendsAsk, question string, def int) int { +// askNumCtx reads a positive integer answer, the default for the empty +// line. Anything else is an error, not a silent fallback: a typo'd window +// ("100k") would otherwise seed a 100000-token prompt budget on a model +// whose window the operator meant to declare as ten times that. +func askNumCtx(ask backendsAsk, question string, def int) (int, error) { answer := strings.TrimSpace(ask.line(question)) if answer == "" { - return def + return def, nil } - if n, err := strconv.Atoi(answer); err == nil && n > 0 { - return n + n, err := strconv.Atoi(answer) + if err != nil || n <= 0 { + return 0, fmt.Errorf("context window %q is not a positive integer (tokens, e.g. 100000) — nothing was written", answer) } - return def + return n, nil } func answerIsYes(answer string) bool { diff --git a/go/internal/cli/init_test.go b/go/internal/cli/init_test.go index 9871d05d..f2f85d47 100644 --- a/go/internal/cli/init_test.go +++ b/go/internal/cli/init_test.go @@ -745,6 +745,7 @@ func TestInitBackendsStepAnthropicAsksEmbedLegSeparately(t *testing.T) { "", // context window → default 100000 "http://gpu:11434", // embedding host "", // embedding protocol → ollama default + "n", // embedding host needs no key }, "sk-ant-live") var ok bool @@ -786,3 +787,24 @@ func TestInitBackendsStepAnthropicAsksEmbedLegSeparately(t *testing.T) { t.Errorf("the Anthropic key must not be attached to the embed row: %v", embed) } } + +// TestInitBackendsStepAnthropicRejectsBadWindow: a non-numeric context +// window aborts the step before anything is sealed or written — a silent +// default would seed the wrong prompt budget. +func TestInitBackendsStepAnthropicRejectsBadWindow(t *testing.T) { + m := newInitMock(t, serverAdmin, nil) + ask, secretCalls := scriptedAsk([]string{ + "https://api.anthropic.com", "anthropic", "claude-opus-5-5", "qwen3-embedding", "100k", + }, "sk-ant-live") + var ok bool + out := captureStdout(t, func() { ok = runBackendsStep(m.client(), ask) }) + if ok { + t.Fatalf("a bad window must fail the step; output:\n%s", out) + } + if !strings.Contains(out, "not a positive integer") { + t.Errorf("output lacks the reason:\n%s", out) + } + if *secretCalls != 0 || len(m.creates) != 0 { + t.Errorf("secret prompts %d, creates %d — want nothing asked or written", *secretCalls, len(m.creates)) + } +} diff --git a/go/internal/handler/backends_manage.go b/go/internal/handler/backends_manage.go index e9f89025..50dbb7cc 100644 --- a/go/internal/handler/backends_manage.go +++ b/go/internal/handler/backends_manage.go @@ -804,8 +804,9 @@ func probeBackendURL(ctx context.Context, b *backends.Backend, checks map[string case anthropic: // The Messages API root answers nothing useful; /v1/models is the // cheapest keyed GET (same root convention: base_url + "/v1/..."). - // A bad key is a 401 here, which the generic branch below reports as - // checks.auth — the chat probe then never runs a priced call. + // A bad key is a 401 here, reported as checks.auth below; the row + // still counts as reachable, so a probe:"chat" goes on to the + // 1-token chat, which 401s the same way — no tokens are billed. url += "/v1/models" case b.ProviderClass == backends.ProviderLlamaCpp: url += "/health" From 2c1d706a78d62c576db8d55e66322288abb7b8cd Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 00:40:04 +0200 Subject: [PATCH 11/16] docs(anthropic): review follow-ups across api, operations, architecture and UI help MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/api.md: the complete stop-reason mapping (stop_sequence, pause_turn, model_context_window_exceeded were applied but not listed), the 16000 max_tokens fallback, the model floor of the always-sent effort field, the mid-stream error classification, the refusal-after-partial-text behaviour, the three session repairs and the two cache breakpoints of the stream path, and the OpenAI-compatible-layer claims marked as what they are — observed on the ku-heimdall install, not reference facts. docs/operations.md: the first-run step names the anthropic wizard questions; the Migration 155 downgrade note describes both old fallthroughs (Ollama path on the pipelines, OpenAI SSE on web chat) and that the pool loads the row unvalidated. docs/architecture.md: the transport-retry enumeration names the anthropic wire. go/internal/config/validate.go: the V4 comment lists the four protocols. Web: ModelMapEditor's help text names the Anthropic wire (and the effort key's destination); settings.ts' dormant protocol mirror and its test carry 'anthropic' so the declared mirror is true again. Co-Authored-By: Claude Fable 5.1 --- docs/api.md | 2 +- docs/architecture.md | 2 +- docs/operations.md | 4 ++-- go/internal/config/validate.go | 2 +- go/web/src/lib/settings.test.ts | 2 +- go/web/src/lib/settings.ts | 2 +- go/web/src/routes/settings/backends/ModelMapEditor.svelte | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/api.md b/docs/api.md index 78728307..76e36e3d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -443,7 +443,7 @@ message — the verdict lives in the body, like every `backend-test` outcome. **OpenRouter (first external backend, G29).** `provider_class: "openrouter"` refines the openai wire: the request **always** carries `provider.zdr:true` + `provider.data_collection:"deny"`, independent of trust level — trust decides WHICH content may flow, the provider class decides whether the provider may store it. `extra_body.provider` entries merge but can only tighten (the force runs after the merge). The single escape is `metadata.allow_data_collection:true`, requiring `confirm_data_collection:true`. Responses feed telemetry: `usage.cost` → llmlog `cost_usd` (local NULL), the top-level `model` (that actually answered) overwrites the row's model, the response `id` lands in `metadata.provider_request_id`. A "no providers" rejection (zdr/deny filter leaves no provider) classifies as configuration-permanent (1h cooldown, no retry storm). `backend-test` additionally reports `credits_remaining`/`usage_usd` (`GET /v1/key`) and `zdr_endpoints` (`GET /v1/endpoints/zdr`). **`base_url` is the API root WITHOUT the version segment** — the wire paths append `/v1/...` themselves; a `base_url` ending in `/v1` double-segments to a 404, and since v4.25.4 create/update reject such a root with a 422 (all openai-, anthropic- and rerank-protocol backends, not just openrouter). -**Anthropic (native Messages API, Migration 155).** `protocol: "anthropic"` speaks `POST {base_url}/v1/messages` directly (`base_url: "https://api.anthropic.com"`, the key from `api_key_ref` travels as `x-api-key`, `anthropic-version: 2023-06-01`) instead of Anthropic's OpenAI-compatible layer — which rejects every sampling parameter on the Claude 5 models, drops `response_format` silently and has neither prompt caching nor effort control. The request carries **no sampling parameters** (`temperature`/`top_p`/`top_k` are 400s on these models; `num_ctx` stays ctx's own prompt budget), the system prompt as a top-level block with `cache_control: ephemeral` (cache hits need a ≥512-token prefix; verify in `metadata.cache_read_input_tokens` of the `context_llm_log` row), `max_tokens` from the pipeline's `num_predict` (**thinking tokens count against it** — the Claude 5 models think adaptively and cannot be told not to, so a model_map `think:false` is ignored on this wire), and `output_config.effort`: the role's `model_map` `params.effort` beats the row's `extra_body.effort`, which beats the explicit default `medium` (the model-side default of Claude Opus 5.5); both carriers accept only `low|medium|high|xhigh|max` (422 otherwise). Every other `extra_body` key merges top-level (the hatch for `thinking.display`, `stop_sequences`, …; the row wins over model_map params on collision, as on the openai wire). **JSON mode is a no-op** on this wire: `output_config.format` (structured outputs) needs a JSON schema with `additionalProperties:false` on every object, which ctx's schemaless `json_object` mode does not have, so `ChatJSON` sends the same body as `Chat` and the pipelines' local parsers stay the validator — run Dream with `CTX_DREAM_JSON_MODE=off` (see [operations](operations.md#environment-variables)). Responses: text blocks are concatenated (thinking blocks skipped), `prompt_tokens` is the **total** prompt (`input_tokens` + `cache_read_input_tokens` + `cache_creation_input_tokens` — Anthropic's `input_tokens` is the uncached remainder only), the two cache counters land in `metadata`, the message id in `metadata.provider_request_id`, `cost_usd` stays NULL (ctx does not price Anthropic; read the console), and `stop_reason` is mapped onto the vocabulary the consumers read (`end_turn`→`stop`, `max_tokens`→`length`, `tool_use`→`tool_calls`, `refusal`→`content_filter`; unknown values pass through raw). Errors classify like every other wire: 429 → rate-limit cooldown (`Retry-After` honoured), 529 `overloaded_error` → gateway (failover), 401 → auth, 400 → request-specific; the provider's error type prefixes the logged message. Row shape (422 on create and update): `api_key_ref` required, `locality: external`, `num_ctx > 0` (no window discovery on this wire — synthesis would fail closed), `provider_class: generic`, no `embed`/`dream-embed`/`rerank` roles (the Messages API has no such endpoints). `backend-test` probes `GET {base_url}/v1/models` with the key, then the 1-token chat. The web-chat stream path (`ChatStreamBackend`) maps the Messages API SSE events (`content_block_delta` text/`input_json_delta`, `message_delta` stop reason + usage, `error`) onto the same event contract as the openai wire, tools included (`tool_use`/`tool_result` blocks, consecutive tool results folded into one user turn). Not available on this wire by construction: logprobs (the dream decide primitive falls back to its generating prompt and memoises the backend as decide-incapable), forced `tool_choice` (a 400 on Claude Opus 5.5; `auto` is what both wires send). +**Anthropic (native Messages API, Migration 155).** `protocol: "anthropic"` speaks `POST {base_url}/v1/messages` directly (`base_url: "https://api.anthropic.com"`, the key from `api_key_ref` travels as `x-api-key`, `anthropic-version: 2023-06-01`) instead of Anthropic's OpenAI-compatible layer — which rejects every sampling parameter on the Claude 5 models (the first production query failed on `temperature`), and, as observed on the ku-heimdall install, drops `response_format` silently and offers neither prompt caching nor effort control. The request carries **no sampling parameters** (`temperature`/`top_p`/`top_k` are 400s on these models; `num_ctx` stays ctx's own prompt budget), the system prompt as a top-level block with `cache_control: ephemeral` (cache hits need a ≥512-token prefix; verify in `metadata.cache_read_input_tokens` of the `context_llm_log` row), `max_tokens` from the pipeline's `num_predict` (16000 when a caller sets none; **thinking tokens count against it** — the Claude 5 models think adaptively and cannot be told not to, so a model_map `think:false` is ignored on this wire; a role whose rows show `metadata.finish_reason = length` needs a larger `model_map` `params.max_tokens` for that role), and `output_config.effort`: the role's `model_map` `params.effort` beats the row's `extra_body.effort`, which beats the explicit default `medium` (the model-side default of Claude Opus 5.5); both carriers accept only `low|medium|high|xhigh|max` (422 otherwise); the field is always sent, so the row needs a model that takes it (the Claude 5 family; Sonnet 4.5 / Haiku 4.5 reject it, and on Opus 4.7/4.8 an omitted `thinking` field means no thinking — neither is a supported row). Every other `extra_body` key merges top-level (the hatch for `thinking.display`, `stop_sequences`, …; the row wins over model_map params on collision, as on the openai wire). **JSON mode is a no-op** on this wire: `output_config.format` (structured outputs) needs a JSON schema with `additionalProperties:false` on every object, which ctx's schemaless `json_object` mode does not have, so `ChatJSON` sends the same body as `Chat` and the pipelines' local parsers stay the validator — run Dream with `CTX_DREAM_JSON_MODE=off` (see [operations](operations.md#environment-variables)). Responses: text blocks are concatenated (thinking blocks skipped), `prompt_tokens` is the **total** prompt (`input_tokens` + `cache_read_input_tokens` + `cache_creation_input_tokens` — Anthropic's `input_tokens` is the uncached remainder only), the two cache counters land in `metadata`, the message id in `metadata.provider_request_id`, `cost_usd` stays NULL (ctx does not price Anthropic; read the console), and `stop_reason` is mapped onto the vocabulary the consumers read (`end_turn`/`stop_sequence`/`pause_turn`→`stop`, `max_tokens`/`model_context_window_exceeded`→`length`, `tool_use`→`tool_calls`, `refusal`→`content_filter`; unknown values pass through raw — a `refusal` after partial text still ends the turn with that text, marked `content_filter` in the `done` event). Errors classify like every other wire: 429 → rate-limit cooldown (`Retry-After` honoured), 529 `overloaded_error` → gateway (failover), 401 → auth, 400 → request-specific, 500 `api_error` → stop (the pool's HTTP-500 doctrine); the provider's error type prefixes the logged message, and the same types inside a mid-stream `error` event (HTTP 200) classify the same way. Row shape (422 on create and update): `api_key_ref` required, `locality: external`, `num_ctx > 0` (no window discovery on this wire — synthesis would fail closed), `provider_class: generic`, no `embed`/`dream-embed`/`rerank` roles (the Messages API has no such endpoints). `backend-test` probes `GET {base_url}/v1/models` with the key, then the 1-token chat. The web-chat stream path (`ChatStreamBackend`) maps the Messages API SSE events (`content_block_delta` text/`input_json_delta`, `message_delta` stop reason + usage, `error`) onto the same event contract as the openai wire, tools included (`tool_use`/`tool_result` blocks, consecutive tool results folded into one user turn). Three repairs keep a session alive on this wire: the answer's `thinking` blocks ride along in memory to the next iteration of the same turn (the API requires the final assistant turn of a tool loop to start with its thinking block; they are not persisted, later turns replay without them), a `tool_use` the history never answered (interrupted turn) gets a synthetic `is_error` `tool_result`, and tool calls that arrived without an id from an openai-wire row earlier in the session get synthetic ids. The system block's breakpoint covers the stable prefix only — the per-call `promptguard` nonce rule goes into a second, uncached block — and the stream path adds the top-level automatic `cache_control` so the growing history is read from cache across iterations and turns. Not available on this wire by construction: logprobs (the dream decide primitive falls back to its generating prompt and memoises the backend as decide-incapable), forced `tool_choice` (a 400 on Claude Opus 5.5; `auto` is what both wires send). ## Web chat sessions diff --git a/docs/architecture.md b/docs/architecture.md index 6aa61c57..0d90e714 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -386,7 +386,7 @@ The table is the **only** source of truth. Nothing seeds it at boot: until an op - **Digest freshness state (retired, migration 129)** — `context_digest_state` plus `trg_digest_dirty`/`mark_digest_dirty()` from the initial schema are gone. The table was verified dead (no Go consumer; the scheduler keeps its dirty flag in memory), but the trigger still fired on **every** block write and updated a SINGLETON row — one shared row lock per writer, i.e. a pure serialisation point at target scale with nothing on the other side of it. Two historical timestamps were dropped with it, deliberately and by explicit decision. Drop order is load-bearing (trigger → function → table): the function resolves its table by name at execution time, so a table dropped first would break not the migration but every subsequent write. - **Supersedes filtering** — temporal-gated removal of outdated blocks from query results. -- **Transport retry** — all inference HTTP calls (chat ollama/openai, embed, rerank) retry exactly once on transient transport failures (connection reset / EOF before any response bytes) via `internal/httpx`. Covers the keep-alive race with llama.cpp's cpp-httplib servers (~5s idle close vs Go connection reuse); HTTP status errors and context deadlines are never retried. Inference POSTs are stateless, so a replay is safe. +- **Transport retry** — all inference HTTP calls (chat ollama/openai/anthropic, embed, rerank) retry exactly once on transient transport failures (connection reset / EOF before any response bytes) via `internal/httpx`. Covers the keep-alive race with llama.cpp's cpp-httplib servers (~5s idle close vs Go connection reuse); HTTP status errors and context deadlines are never retried. Inference POSTs are stateless, so a replay is safe. - **Streaming tool-call wire** (`llm.ChatStream`) — streaming OpenAI-compatible chat with function calling: multi-turn message arrays, per-delta events, index-keyed tool-call assembly, arguments normalisation (llama.cpp JSON-string fragments and whole-object form yield identical calls), hardened against OpenRouter SSE comment frames and mid-stream error events inside HTTP-200 streams; usage falls back to llama.cpp `timings` incl. MTP draft-acceptance. Drives the web-chat harness (see [api](api.md#web-chat-sessions)). Since issue #26 the **non-streaming** chat path carries the provider stop reason too (`ChatResponse.FinishReason`, decoded from Ollama `done_reason` resp. OpenAI `choices[0].finish_reason`, raw and unnormalised, `""` when the provider reports none) — until then only the SSE path saw it, which left every background pipeline unable to tell a `length` truncation from malformed output. - **Decide primitive** (`llm.DecideChoice`, `llm.DecideOptions`) — a classification read off the FIRST generated token's top-logprobs instead of decoded text (System-One style). The caller phrases its question so the answer is one short label token (`A`…`E`, `Yes`/`No`); the OpenAI wire is asked for `logprobs`/`top_logprobs` (`Options.TopLogprobs`, JSON-tagged off so the Ollama options block never carries it), and `ChatResponse.TopLogprobs` hands the alternatives back as a distribution over the labels — renormalised probabilities, the raw label **mass** (share of belief that landed in the requested vocabulary at all) and a TypeSafe-style **confidence** `(n·peak−1)/(n−1)`. Rationale: on the serving hardware one output token costs ~100× an input token (decode 18–30 tok/s vs prefill ≈2.5k tok/s) and a decision needs none; there is also no JSON to truncate or drift. A backend that reports no logprobs (Ollama, some relays) or an answer outside the label set (`ErrNoLogprobs` / `ErrNoLabelMass`, `IsDecideFallback`) is a fallback signal, never a fabricated distribution. Measured 2026-09-20 against the goldbench gold sets: rerank nDCG@15 0.841 vs 0.648 for the generating judge (equal on the judge's parsed cases, no parse failures), dream links 0.605 vs 0.599, recurrence 0.885 = 0.885, at 0 output tokens. - **Embed cache** — content-hash-keyed embedding cache (`context_embed_cache`) to avoid re-embedding identical text across pipelines. diff --git a/docs/operations.md b/docs/operations.md index e56937c3..69b5200b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -10,7 +10,7 @@ The wizard is the documented first-run path and is idempotent — it reports eve 1. **Config** — asks for base URL and API key, writes `~/.config/ctx/config` (mode 0600). 2. **Server** — reads `/health`. A `degraded` or `unhealthy` server is shown but accepted: an install whose [backend pool](#backends) is still empty answers `unhealthy` with HTTP 503 by design, and that is precisely the state the next step fixes. -3. **Backend pool** — lists the pool and, when no enabled `_global` row serves the `synthesis` or `embed` role, asks for host, protocol, chat model, embedding model and an optional API key, then runs the same seed `ctx backends seed` runs (same server-admin gate, same full-trust posture, same sealed-secret handling — the key is read without echo and never lands in the row). Afterwards, and whenever it finds a pool already populated, it **probes** each serving row and names the dead ones. A key below server-admin is not an error here: the step says so, points at `ctx backends seed`, and init carries on — hooks and statusline do not need an admin key. +3. **Backend pool** — lists the pool and, when no enabled `_global` row serves the `synthesis` or `embed` role, asks for host, protocol, chat model, embedding model and an optional API key (for the `anthropic` protocol: the chat model's context window, a separate embedding host and protocol, and the key), then runs the same seed `ctx backends seed` runs (same server-admin gate, same full-trust posture, same sealed-secret handling — the key is read without echo and never lands in the row). Afterwards, and whenever it finds a pool already populated, it **probes** each serving row and names the dead ones. A key below server-admin is not an error here: the step says so, points at `ctx backends seed`, and init carries on — hooks and statusline do not need an admin key. 4. **Version** — compares the local build against the latest GitHub release. 5. **Hooks** — `SubagentStart` / `SubagentStop` in Claude Code's `settings.json`. 6. **Statusline** — `ctx statusline` in the same file. @@ -725,7 +725,7 @@ docker exec n8n-db-1 psql -U "$CONTEXT_DB_USER" -d "$CONTEXT_DB" -c \ ### Migration 155: `anthropic` joins the backend wire protocols -`context_backends.protocol` was pinned by a `CHECK` to `openai`, `ollama` and `rerank`. Migration 155 widens it to include `anthropic` (the native Messages API wire, [api — Anthropic](api.md#backend-pool)). It resolves the baseline's unnamed constraint from `pg_constraint` by table and definition, drops it and re-adds it as `context_backends_protocol_check`; no row changes, the column keeps its `DEFAULT 'openai'`. Nothing to do at upgrade time. **Downgrading** a daemon below this migration with an `anthropic` row still in the pool makes that row dispatch onto the Ollama wire (the old fallthrough for unknown protocols) — disable or delete such rows before rolling the binary back; the widened `CHECK` itself is harmless to an older binary. +`context_backends.protocol` was pinned by a `CHECK` to `openai`, `ollama` and `rerank`. Migration 155 widens it to include `anthropic` (the native Messages API wire, [api — Anthropic](api.md#backend-pool)). It resolves the baseline's unnamed constraint from `pg_constraint` by table and definition, drops it and re-adds it as `context_backends_protocol_check`; no row changes, the column keeps its `DEFAULT 'openai'`. Nothing to do at upgrade time. **Downgrading** a daemon below this migration with an `anthropic` row still in the pool makes that row dispatch onto the wrong wire — the Ollama `/api/chat` path on the pipelines (the old fallthrough for unknown protocols), the OpenAI SSE path on web chat (the old stream path never read the protocol); the pool loads the row either way, validation runs only on create and update — so disable or delete such rows before rolling the binary back; the widened `CHECK` itself is harmless to an older binary. ### `graph_overview.csr_loader`: the rebuild's input substrate diff --git a/go/internal/config/validate.go b/go/internal/config/validate.go index 2870b917..2cf804b8 100644 --- a/go/internal/config/validate.go +++ b/go/internal/config/validate.go @@ -790,7 +790,7 @@ func validateIntSigns(c *Config) []Issue { // - V4 (protocol typos: anything but ollama/openai fell silently onto the // ollama wire path → 404 against llama.cpp) is enforced at the same // boundary: validateIdentity rejects a row whose protocol is not one of -// openai, ollama, rerank. +// openai, ollama, anthropic, rerank. // - V1 (dual-runner VRAM WARN, β6), V12 (dream-embed credential boundary, // β5), V8 and V13 (rerank host reachability WARNs, β3) are NOT rebuildable // here and are not rebuilt: each compared fields of two tuples, and which diff --git a/go/web/src/lib/settings.test.ts b/go/web/src/lib/settings.test.ts index 17c8651c..e308ec1b 100644 --- a/go/web/src/lib/settings.test.ts +++ b/go/web/src/lib/settings.test.ts @@ -41,7 +41,7 @@ describe('widgetFor', () => { describe('selectOptions', () => { it('mirrors the Go enums', () => { - expect(selectOptions('protocol')).toEqual(['ollama', 'openai']) + expect(selectOptions('protocol')).toEqual(['ollama', 'openai', 'anthropic']) expect(selectOptions('think')).toEqual(['true', 'false']) expect(selectOptions('string')).toEqual([]) }) diff --git a/go/web/src/lib/settings.ts b/go/web/src/lib/settings.ts index b437515f..df332666 100644 --- a/go/web/src/lib/settings.ts +++ b/go/web/src/lib/settings.ts @@ -43,7 +43,7 @@ export function widgetFor(type: string): Widget { export function selectOptions(type: string): string[] { switch (type) { case 'protocol': - return ['ollama', 'openai'] + return ['ollama', 'openai', 'anthropic'] case 'think': return ['true', 'false'] case 'sensitivity': diff --git a/go/web/src/routes/settings/backends/ModelMapEditor.svelte b/go/web/src/routes/settings/backends/ModelMapEditor.svelte index 55868d8e..294d6c58 100644 --- a/go/web/src/routes/settings/backends/ModelMapEditor.svelte +++ b/go/web/src/routes/settings/backends/ModelMapEditor.svelte @@ -134,7 +134,7 @@ {#if invalid[i]} invalid JSON — last valid state is kept {:else} - JSON object, sent verbatim as model_map params (empty = none). Unknown keys reach the wire on OpenAI-protocol backends. + JSON object, sent verbatim as model_map params (empty = none). Unknown keys reach the wire on OpenAI- and Anthropic-protocol backends (on Anthropic, effort becomes output_config.effort). {/if}

From c147fe4dbb64b197d92864f2aecc6f406e553e87 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 12:37:13 +0200 Subject: [PATCH 12/16] ci(lint): bump golangci-lint to v2.14.0 for the go1.27.2 runner toolchain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Lint job fails on every run since 2026-10-10 (nightly on root 38019738113 and 38107728729, this PR's 38132148410), all other jobs green: GO_VERSION "1.27" now resolves to go1.27.2 on the runner, and the pinned v2.13.2 cannot read its export data — "export data version 5 is greater than maximum supported version 4", reported as a typecheck error on the first `import "fmt"` (internal/clustersql/clustersql.go:28). v2.14.0 (2026-09-24) reads it; locally against go1.27.2 with the same .golangci.yml it reports 0 issues over ./..., and the deadcode gate is green. Deliberate pin bump, as the comment in the workflow asks. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ab33c7a..c77927f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,16 @@ jobs: # der es selbst gebaut wurde, und v2.11.4 bricht gegen ein Modul mit # `go 1.27.0` mit "the Go language version used to build # golangci-lint is lower than the targeted Go version" ab. - version: v2.13.2 + # + # Bump auf v2.14.0 (2026-10-11, PR #47): der Runner löst GO_VERSION + # "1.27" seit dem 10.10. auf go1.27.2 auf, und v2.13.2 (gebaut mit + # einem älteren 1.27) kann dessen Export-Daten nicht lesen — + # "export data version 5 is greater than maximum supported version + # 4", typecheck-Fehler auf dem ersten `import "fmt"`, Nightly auf + # root rot seit 38019738113 (10.10.) bei sonst grünen Jobs. v2.14.0 + # (24.09.) liest sie; lokal gegen go1.27.2 mit demselben + # .golangci.yml: 0 issues. + version: v2.14.0 working-directory: go args: --timeout=5m # Kein Analyse-Cache (2026-08-26): der Cache-Key der Action trägt das From 97e0821996bdfadbe2b28641cc3bc9138097e195 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 12:37:37 +0200 Subject: [PATCH 13/16] test(llm): env-gated live smoke of the anthropic wire against api.anthropic.com Skipped under -short and without CTX_ANTHROPIC_SMOKE_KEY, so CI never spends tokens. By hand it proves what the httptest fakes cannot: the request shape is accepted by the real endpoint, the prompt cache hits on the second call (stable prefix above 512 tokens, fresh promptguard nonce per call), the thinking block replays through a streamed tool round trip (and what the API says about the same round trip without it), effort low and high are accepted, the 1-token backend-test probe's behaviour on a thinking model, and a bad key classifying as auth. Every probe logs the usage it saw (tokens, cache counters, message id). Co-Authored-By: Claude Fable 5.1 --- go/internal/llm/anthropic_live_test.go | 197 +++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 go/internal/llm/anthropic_live_test.go diff --git a/go/internal/llm/anthropic_live_test.go b/go/internal/llm/anthropic_live_test.go new file mode 100644 index 00000000..3b096e30 --- /dev/null +++ b/go/internal/llm/anthropic_live_test.go @@ -0,0 +1,197 @@ +// Live smoke of the anthropic wire against api.anthropic.com. Skipped under +// -short and without CTX_ANTHROPIC_SMOKE_KEY, so CI never spends tokens; run +// it by hand from a shell that holds the key: +// +// CTX_ANTHROPIC_SMOKE_KEY=sk-ant-… go test ./internal/llm/ -run Live -count=1 -v +// +// It proves what the fakes cannot: the request shape is accepted by the real +// endpoint, the prompt cache hits on the second call, the thinking block is +// replayed correctly through a tool round trip, effort is accepted, the +// 1-token probe's behaviour, and the 401 classification. Each probe logs the +// usage it saw. Cost: a few thousand tokens per run. +package llm + +import ( + "context" + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/GottZ/ctx/internal/backends" + "github.com/GottZ/ctx/internal/httpx" + "github.com/GottZ/ctx/internal/promptguard" +) + +func liveAnthropicBackend(t *testing.T) backends.Backend { + t.Helper() + if testing.Short() { + t.Skip("live smoke skipped under -short") + } + key := os.Getenv("CTX_ANTHROPIC_SMOKE_KEY") + if key == "" { + t.Skip("CTX_ANTHROPIC_SMOKE_KEY not set") + } + model := os.Getenv("CTX_ANTHROPIC_SMOKE_MODEL") + if model == "" { + model = "claude-opus-5-5" + } + host := os.Getenv("CTX_ANTHROPIC_SMOKE_HOST") + if host == "" { + host = "https://api.anthropic.com" + } + return backends.Backend{ + Host: host, Protocol: backends.ProtocolAnthropic, ProviderClass: backends.ProviderGeneric, + Model: model, Trust: backends.TrustNoCredentials, Locality: backends.LocalityExternal, + APIKey: key, APIKeyRef: "smoke", NumCtx: 100000, + ModelMap: map[string]backends.ModelSpec{"default": {Model: model}}, + } +} + +// liveSystemPrompt is a stable prefix comfortably above the 512-token cache +// minimum, followed by the per-call promptguard rule exactly as the +// pipelines send it. +func liveSystemPrompt() string { + var sb strings.Builder + sb.WriteString("You are ctx, a knowledge store assistant. Answer briefly and only from the sources given. ") + for i := 0; i < 60; i++ { + sb.WriteString("Rule ") + sb.WriteString(strings.Repeat("alpha beta gamma delta ", 3)) + sb.WriteString("applies to every answer; cite the source id in brackets. ") + } + sb.WriteString(promptguard.Rule(promptguard.NewNonce())) + return sb.String() +} + +func TestAnthropicLiveNonStreamAndCache(t *testing.T) { + b := liveAnthropicBackend(t) + ctx := context.Background() + opts := SynthesisOptions(100000) // the production preset: temperature, repeat penalty (both dropped), NumPredict 500 + first, err := Chat(ctx, b, liveSystemPrompt(), "Reply with exactly: ok", opts, 60*time.Second) + if err != nil { + t.Fatalf("first call: %v", err) + } + t.Logf("first: finish=%q out=%d prompt=%d cache_read=%d cache_create=%d id=%s content=%q", + first.FinishReason, first.EvalCount, first.PromptTokens, first.CacheReadTokens, first.CacheCreationTokens, first.ProviderRequestID, first.Message.Content) + if first.Message.Content == "" || first.EvalCount == 0 { + t.Errorf("empty answer: %+v", first) + } + second, err := Chat(ctx, b, liveSystemPrompt(), "Reply with exactly: ok again", opts, 60*time.Second) + if err != nil { + t.Fatalf("second call: %v", err) + } + t.Logf("second: finish=%q out=%d prompt=%d cache_read=%d cache_create=%d", + second.FinishReason, second.EvalCount, second.PromptTokens, second.CacheReadTokens, second.CacheCreationTokens) + if second.CacheReadTokens == 0 { + t.Errorf("no cache hit on the second call with a byte-identical stable prefix (fresh nonce only): cache_read=0, cache_create=%d", second.CacheCreationTokens) + } + + // JSON mode is a no-op on this wire: the same call must still succeed. + j, err := ChatJSON(ctx, b, "Answer with a JSON object {\"ok\": true}.", "Go.", Options{NumPredict: 200}, 60*time.Second) + if err != nil { + t.Fatalf("ChatJSON: %v", err) + } + t.Logf("json-mode no-op: finish=%q content=%q", j.FinishReason, j.Message.Content) +} + +func TestAnthropicLiveEffortAndProbe(t *testing.T) { + b := liveAnthropicBackend(t) + ctx := context.Background() + for _, eff := range []string{"low", "high"} { + r, err := Chat(ctx, b, "Answer in one word.", "What colour is the sky on a clear day?", + Options{NumPredict: 300, Extra: map[string]any{"effort": eff}}, 60*time.Second) + if err != nil { + t.Fatalf("effort %s: %v", eff, err) + } + t.Logf("effort=%s finish=%q out=%d content=%q", eff, r.FinishReason, r.EvalCount, r.Message.Content) + } + // The backend-test chat probe shape: max_tokens 1 on a model that thinks. + p, err := Chat(ctx, b, "You are a connectivity probe.", "Reply with: ok", Options{Temperature: 0, NumPredict: 1}, 30*time.Second) + if err != nil { + var se *httpx.StatusError + if errors.As(err, &se) { + t.Logf("probe max_tokens=1: HTTP %d %s (the probe would report %s)", se.Code, se.Body, backends.Classify(err, backends.ProviderGeneric)) + } else { + t.Fatalf("probe: %v", err) + } + } else { + t.Logf("probe max_tokens=1: finish=%q out=%d content=%q", p.FinishReason, p.EvalCount, p.Message.Content) + } +} + +func TestAnthropicLiveStreamToolRoundTrip(t *testing.T) { + b := liveAnthropicBackend(t) + ctx := context.Background() + tool := ToolDef{Type: "function", Function: ToolDefFunction{ + Name: "get_weather", Description: "Current weather for a city", + Parameters: json.RawMessage(`{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}`), + }} + msgs := []ChatMsg{ + {Role: "system", Content: "You are a weather assistant. Use the get_weather tool to answer weather questions; never guess."}, + {Role: "user", Content: "What is the weather in Berlin right now? Use the tool."}, + } + var events []StreamEvent + res, err := ChatStreamBackend(ctx, b, b.Model, msgs, []ToolDef{tool}, Options{NumPredict: 1024}, 90*time.Second, collectEvents(&events)) + if err != nil { + t.Fatalf("first stream: %v", err) + } + t.Logf("first stream: finish=%q calls=%d blocks=%d out=%d prompt=%d cache_read=%d cache_create=%d content=%q", + res.FinishReason, len(res.ToolCalls), len(res.ProviderBlocks), res.CompletionTokens, res.PromptTokens, res.CacheReadTokens, res.CacheCreationTokens, res.Content) + if res.FinishReason != "tool_calls" || len(res.ToolCalls) == 0 { + t.Fatalf("expected a tool call, got finish=%q calls=%+v", res.FinishReason, res.ToolCalls) + } + for _, blk := range res.ProviderBlocks { + t.Logf("provider block: %.120s", blk) + } + call := res.ToolCalls[0] + args := argsAsMap(t, call) + t.Logf("tool call: id=%s name=%s args=%v", call.ID, call.Function.Name, args) + + // Round trip: the assistant turn WITH its thinking block, then the result. + msgs = append(msgs, + ChatMsg{Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls, ProviderBlocks: res.ProviderBlocks}, + ChatMsg{Role: "tool", Content: `{"temperature_c": 21, "sky": "clear"}`, ToolCallID: call.ID}, + ) + events = events[:0] + final, err := ChatStreamBackend(ctx, b, b.Model, msgs, []ToolDef{tool}, Options{NumPredict: 1024}, 90*time.Second, collectEvents(&events)) + if err != nil { + t.Fatalf("second stream (thinking replay): %v", err) + } + t.Logf("second stream: finish=%q out=%d prompt=%d cache_read=%d cache_create=%d content=%q", + final.FinishReason, final.CompletionTokens, final.PromptTokens, final.CacheReadTokens, final.CacheCreationTokens, final.Content) + if final.FinishReason != "stop" || !strings.Contains(final.Content, "21") { + t.Errorf("expected a final answer quoting the tool result, got finish=%q content=%q", final.FinishReason, final.Content) + } + if final.CacheReadTokens == 0 { + t.Logf("note: no cache read on the second iteration (prefix below the 512-token minimum is expected for this tiny prompt)") + } + + // The same round trip WITHOUT the thinking block is what the wire did + // before the review fix — record what the API says about it today. + noThinking := append([]ChatMsg{}, msgs[:2]...) + noThinking = append(noThinking, + ChatMsg{Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls}, + ChatMsg{Role: "tool", Content: `{"temperature_c": 21, "sky": "clear"}`, ToolCallID: call.ID}, + ) + if _, err := ChatStreamBackend(ctx, b, b.Model, noThinking, []ToolDef{tool}, Options{NumPredict: 256}, 90*time.Second, nil); err != nil { + t.Logf("round trip without the thinking block: %v (expected on the Claude 5 models — this is why ProviderBlocks exist)", err) + } else { + t.Logf("round trip without the thinking block: accepted by the API today") + } +} + +func TestAnthropicLiveBadKeyClassifiesAuth(t *testing.T) { + b := liveAnthropicBackend(t) + b.APIKey = "sk-ant-invalid-smoke" + _, err := Chat(context.Background(), b, "s", "Reply with: ok", Options{NumPredict: 5}, 30*time.Second) + if err == nil { + t.Fatal("a bad key must fail") + } + cls := backends.Classify(err, backends.ProviderGeneric) + t.Logf("bad key: %v → %s", err, cls) + if cls != backends.ClassAuth { + t.Errorf("Classify = %s, want auth", cls) + } +} From e0c79bb0b7916872921894cc5cbcd46fbc2e5466 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 12:39:48 +0200 Subject: [PATCH 14/16] test(backends): pin Migration 155's protocol CHECK on a real PostgreSQL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integration-tagged: the full migration chain on the prod-parity image, then the constraint context_backends_protocol_check exists by name, admits 'anthropic' next to openai/ollama/rerank, is the only CHECK on the table that mentions the column (the DO block's catalog lookup stays unambiguous), an anthropic row inserts, and an unknown protocol fails with SQLSTATE 23514 on that named constraint. Ran locally against testcontainers (colima) — PASS in 2.2 s; the schema-manifest generator run on the same image reproduced the hand-bumped manifest.json byte for byte. Co-Authored-By: Claude Fable 5.1 --- .../protocol_anthropic_integration_test.go | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 go/internal/backends/protocol_anthropic_integration_test.go diff --git a/go/internal/backends/protocol_anthropic_integration_test.go b/go/internal/backends/protocol_anthropic_integration_test.go new file mode 100644 index 00000000..61d1b2e4 --- /dev/null +++ b/go/internal/backends/protocol_anthropic_integration_test.go @@ -0,0 +1,61 @@ +//go:build integration + +package backends_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + + "github.com/GottZ/ctx/internal/testdb" +) + +// TestMigration155ProtocolCheckAdmitsAnthropic runs the real migration chain +// on a prod-parity PostgreSQL and pins what Migration 155 changed: the +// (formerly unnamed) CHECK on context_backends.protocol now carries its +// explicit name, admits 'anthropic' next to the three original wires, and +// still rejects anything else with a check_violation (SQLSTATE 23514) — so a +// typo'd protocol cannot reach the pool and fall onto the Ollama path. +func TestMigration155ProtocolCheckAdmitsAnthropic(t *testing.T) { + pool := testdb.SetupTestDB(t) + ctx := context.Background() + + var def string + if err := pool.QueryRow(ctx, + `SELECT pg_get_constraintdef(oid) FROM pg_constraint + WHERE conrelid = 'context_backends'::regclass AND conname = 'context_backends_protocol_check'`, + ).Scan(&def); err != nil { + t.Fatalf("constraint context_backends_protocol_check not found after the chain: %v", err) + } + for _, want := range []string{"openai", "ollama", "rerank", "anthropic"} { + if !strings.Contains(def, "'"+want+"'") { + t.Errorf("constraint %q lacks %q", def, want) + } + } + + // Exactly one CHECK mentions the column — the DO block's catalog lookup + // stays unambiguous. + var n int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM pg_constraint + WHERE conrelid = 'context_backends'::regclass AND contype = 'c' + AND pg_get_constraintdef(oid) LIKE '%protocol%'`).Scan(&n); err != nil || n != 1 { + t.Errorf("CHECKs mentioning protocol = %d (err %v), want exactly 1", n, err) + } + + if _, err := pool.Exec(ctx, + `INSERT INTO context_backends (name, base_url, protocol, api_key_ref, locality, num_ctx) + VALUES ('anthropic-it', 'https://api.anthropic.com', 'anthropic', 'anthropic-chat', 'external', 100000)`); err != nil { + t.Fatalf("anthropic row rejected by the widened CHECK: %v", err) + } + + _, err := pool.Exec(ctx, + `INSERT INTO context_backends (name, base_url, protocol) VALUES ('grpc-it', 'http://x', 'grpc')`) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "23514" || pgErr.ConstraintName != "context_backends_protocol_check" { + t.Fatalf("unknown protocol must fail the named CHECK with 23514, got %v", err) + } +} From 1ccbe733386488790fc0618356ede9e5a72d9382 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 12:50:55 +0200 Subject: [PATCH 15/16] fix(llm): state the thinking-replay rule as the reference's, not as an observed 400 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adversarial review of the last three commits caught an overstatement: docs/api.md, the anthropic_stream.go comments and a test comment said the API "requires" the final assistant turn of a tool loop to start with its thinking block. The claude-api reference asks that a turn whose tool_use still awaits its result KEEP its thinking block and rejects an altered one; live on 2026-10-11 (claude-opus-5-5) a short tool turn came back with no thinking block at all — even with thinking.display summarized and effort high — and the round trip was accepted. The replay mechanism stays (it honours the reference's rule whenever a block is produced, and the fixtures cover that path); the wording now says exactly that. The live smoke test reports when the replay path was not exercised instead of logging an "accepted without the block" line that compared two identical requests, asks for visible reasoning on the tool turn, and its header says that the integration job (which runs without -short) relies on the absent key alone. The Migration 155 integration test also pins that the column's DEFAULT 'openai' survives the constraint swap. Co-Authored-By: Claude Fable 5.1 --- docs/api.md | 2 +- .../protocol_anthropic_integration_test.go | 8 +++++++ go/internal/llm/anthropic_live_test.go | 24 ++++++++++++++----- go/internal/llm/anthropic_stream.go | 15 ++++++++---- go/internal/llm/anthropic_stream_test.go | 4 ++-- 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/docs/api.md b/docs/api.md index 76e36e3d..234598b1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -443,7 +443,7 @@ message — the verdict lives in the body, like every `backend-test` outcome. **OpenRouter (first external backend, G29).** `provider_class: "openrouter"` refines the openai wire: the request **always** carries `provider.zdr:true` + `provider.data_collection:"deny"`, independent of trust level — trust decides WHICH content may flow, the provider class decides whether the provider may store it. `extra_body.provider` entries merge but can only tighten (the force runs after the merge). The single escape is `metadata.allow_data_collection:true`, requiring `confirm_data_collection:true`. Responses feed telemetry: `usage.cost` → llmlog `cost_usd` (local NULL), the top-level `model` (that actually answered) overwrites the row's model, the response `id` lands in `metadata.provider_request_id`. A "no providers" rejection (zdr/deny filter leaves no provider) classifies as configuration-permanent (1h cooldown, no retry storm). `backend-test` additionally reports `credits_remaining`/`usage_usd` (`GET /v1/key`) and `zdr_endpoints` (`GET /v1/endpoints/zdr`). **`base_url` is the API root WITHOUT the version segment** — the wire paths append `/v1/...` themselves; a `base_url` ending in `/v1` double-segments to a 404, and since v4.25.4 create/update reject such a root with a 422 (all openai-, anthropic- and rerank-protocol backends, not just openrouter). -**Anthropic (native Messages API, Migration 155).** `protocol: "anthropic"` speaks `POST {base_url}/v1/messages` directly (`base_url: "https://api.anthropic.com"`, the key from `api_key_ref` travels as `x-api-key`, `anthropic-version: 2023-06-01`) instead of Anthropic's OpenAI-compatible layer — which rejects every sampling parameter on the Claude 5 models (the first production query failed on `temperature`), and, as observed on the ku-heimdall install, drops `response_format` silently and offers neither prompt caching nor effort control. The request carries **no sampling parameters** (`temperature`/`top_p`/`top_k` are 400s on these models; `num_ctx` stays ctx's own prompt budget), the system prompt as a top-level block with `cache_control: ephemeral` (cache hits need a ≥512-token prefix; verify in `metadata.cache_read_input_tokens` of the `context_llm_log` row), `max_tokens` from the pipeline's `num_predict` (16000 when a caller sets none; **thinking tokens count against it** — the Claude 5 models think adaptively and cannot be told not to, so a model_map `think:false` is ignored on this wire; a role whose rows show `metadata.finish_reason = length` needs a larger `model_map` `params.max_tokens` for that role), and `output_config.effort`: the role's `model_map` `params.effort` beats the row's `extra_body.effort`, which beats the explicit default `medium` (the model-side default of Claude Opus 5.5); both carriers accept only `low|medium|high|xhigh|max` (422 otherwise); the field is always sent, so the row needs a model that takes it (the Claude 5 family; Sonnet 4.5 / Haiku 4.5 reject it, and on Opus 4.7/4.8 an omitted `thinking` field means no thinking — neither is a supported row). Every other `extra_body` key merges top-level (the hatch for `thinking.display`, `stop_sequences`, …; the row wins over model_map params on collision, as on the openai wire). **JSON mode is a no-op** on this wire: `output_config.format` (structured outputs) needs a JSON schema with `additionalProperties:false` on every object, which ctx's schemaless `json_object` mode does not have, so `ChatJSON` sends the same body as `Chat` and the pipelines' local parsers stay the validator — run Dream with `CTX_DREAM_JSON_MODE=off` (see [operations](operations.md#environment-variables)). Responses: text blocks are concatenated (thinking blocks skipped), `prompt_tokens` is the **total** prompt (`input_tokens` + `cache_read_input_tokens` + `cache_creation_input_tokens` — Anthropic's `input_tokens` is the uncached remainder only), the two cache counters land in `metadata`, the message id in `metadata.provider_request_id`, `cost_usd` stays NULL (ctx does not price Anthropic; read the console), and `stop_reason` is mapped onto the vocabulary the consumers read (`end_turn`/`stop_sequence`/`pause_turn`→`stop`, `max_tokens`/`model_context_window_exceeded`→`length`, `tool_use`→`tool_calls`, `refusal`→`content_filter`; unknown values pass through raw — a `refusal` after partial text still ends the turn with that text, marked `content_filter` in the `done` event). Errors classify like every other wire: 429 → rate-limit cooldown (`Retry-After` honoured), 529 `overloaded_error` → gateway (failover), 401 → auth, 400 → request-specific, 500 `api_error` → stop (the pool's HTTP-500 doctrine); the provider's error type prefixes the logged message, and the same types inside a mid-stream `error` event (HTTP 200) classify the same way. Row shape (422 on create and update): `api_key_ref` required, `locality: external`, `num_ctx > 0` (no window discovery on this wire — synthesis would fail closed), `provider_class: generic`, no `embed`/`dream-embed`/`rerank` roles (the Messages API has no such endpoints). `backend-test` probes `GET {base_url}/v1/models` with the key, then the 1-token chat. The web-chat stream path (`ChatStreamBackend`) maps the Messages API SSE events (`content_block_delta` text/`input_json_delta`, `message_delta` stop reason + usage, `error`) onto the same event contract as the openai wire, tools included (`tool_use`/`tool_result` blocks, consecutive tool results folded into one user turn). Three repairs keep a session alive on this wire: the answer's `thinking` blocks ride along in memory to the next iteration of the same turn (the API requires the final assistant turn of a tool loop to start with its thinking block; they are not persisted, later turns replay without them), a `tool_use` the history never answered (interrupted turn) gets a synthetic `is_error` `tool_result`, and tool calls that arrived without an id from an openai-wire row earlier in the session get synthetic ids. The system block's breakpoint covers the stable prefix only — the per-call `promptguard` nonce rule goes into a second, uncached block — and the stream path adds the top-level automatic `cache_control` so the growing history is read from cache across iterations and turns. Not available on this wire by construction: logprobs (the dream decide primitive falls back to its generating prompt and memoises the backend as decide-incapable), forced `tool_choice` (a 400 on Claude Opus 5.5; `auto` is what both wires send). +**Anthropic (native Messages API, Migration 155).** `protocol: "anthropic"` speaks `POST {base_url}/v1/messages` directly (`base_url: "https://api.anthropic.com"`, the key from `api_key_ref` travels as `x-api-key`, `anthropic-version: 2023-06-01`) instead of Anthropic's OpenAI-compatible layer — which rejects every sampling parameter on the Claude 5 models (the first production query failed on `temperature`), and, as observed on the ku-heimdall install, drops `response_format` silently and offers neither prompt caching nor effort control. The request carries **no sampling parameters** (`temperature`/`top_p`/`top_k` are 400s on these models; `num_ctx` stays ctx's own prompt budget), the system prompt as a top-level block with `cache_control: ephemeral` (cache hits need a ≥512-token prefix; verify in `metadata.cache_read_input_tokens` of the `context_llm_log` row), `max_tokens` from the pipeline's `num_predict` (16000 when a caller sets none; **thinking tokens count against it** — the Claude 5 models think adaptively and cannot be told not to, so a model_map `think:false` is ignored on this wire; a role whose rows show `metadata.finish_reason = length` needs a larger `model_map` `params.max_tokens` for that role), and `output_config.effort`: the role's `model_map` `params.effort` beats the row's `extra_body.effort`, which beats the explicit default `medium` (the model-side default of Claude Opus 5.5); both carriers accept only `low|medium|high|xhigh|max` (422 otherwise); the field is always sent, so the row needs a model that takes it (the Claude 5 family; Sonnet 4.5 / Haiku 4.5 reject it, and on Opus 4.7/4.8 an omitted `thinking` field means no thinking — neither is a supported row). Every other `extra_body` key merges top-level (the hatch for `thinking.display`, `stop_sequences`, …; the row wins over model_map params on collision, as on the openai wire). **JSON mode is a no-op** on this wire: `output_config.format` (structured outputs) needs a JSON schema with `additionalProperties:false` on every object, which ctx's schemaless `json_object` mode does not have, so `ChatJSON` sends the same body as `Chat` and the pipelines' local parsers stay the validator — run Dream with `CTX_DREAM_JSON_MODE=off` (see [operations](operations.md#environment-variables)). Responses: text blocks are concatenated (thinking blocks skipped), `prompt_tokens` is the **total** prompt (`input_tokens` + `cache_read_input_tokens` + `cache_creation_input_tokens` — Anthropic's `input_tokens` is the uncached remainder only), the two cache counters land in `metadata`, the message id in `metadata.provider_request_id`, `cost_usd` stays NULL (ctx does not price Anthropic; read the console), and `stop_reason` is mapped onto the vocabulary the consumers read (`end_turn`/`stop_sequence`/`pause_turn`→`stop`, `max_tokens`/`model_context_window_exceeded`→`length`, `tool_use`→`tool_calls`, `refusal`→`content_filter`; unknown values pass through raw — a `refusal` after partial text still ends the turn with that text, marked `content_filter` in the `done` event). Errors classify like every other wire: 429 → rate-limit cooldown (`Retry-After` honoured), 529 `overloaded_error` → gateway (failover), 401 → auth, 400 → request-specific, 500 `api_error` → stop (the pool's HTTP-500 doctrine); the provider's error type prefixes the logged message, and the same types inside a mid-stream `error` event (HTTP 200) classify the same way. Row shape (422 on create and update): `api_key_ref` required, `locality: external`, `num_ctx > 0` (no window discovery on this wire — synthesis would fail closed), `provider_class: generic`, no `embed`/`dream-embed`/`rerank` roles (the Messages API has no such endpoints). `backend-test` probes `GET {base_url}/v1/models` with the key, then the 1-token chat. The web-chat stream path (`ChatStreamBackend`) maps the Messages API SSE events (`content_block_delta` text/`input_json_delta`, `message_delta` stop reason + usage, `error`) onto the same event contract as the openai wire, tools included (`tool_use`/`tool_result` blocks, consecutive tool results folded into one user turn). Three repairs keep a session alive on this wire: the answer's `thinking` blocks, when the model emits any, ride along in memory to the next iteration of the same turn and are replayed verbatim ahead of the `tool_use` they introduced (the reference asks that a turn whose `tool_use` still awaits its result keep its thinking block, and an altered block is rejected; live on 2026-10-11 a short tool turn carried no thinking block and the round trip was accepted — they are not persisted, later turns replay without them), a `tool_use` the history never answered (interrupted turn) gets a synthetic `is_error` `tool_result`, and tool calls that arrived without an id from an openai-wire row earlier in the session get synthetic ids. The system block's breakpoint covers the stable prefix only — the per-call `promptguard` nonce rule goes into a second, uncached block — and the stream path adds the top-level automatic `cache_control` so the growing history is read from cache across iterations and turns. Not available on this wire by construction: logprobs (the dream decide primitive falls back to its generating prompt and memoises the backend as decide-incapable), forced `tool_choice` (a 400 on Claude Opus 5.5; `auto` is what both wires send). ## Web chat sessions diff --git a/go/internal/backends/protocol_anthropic_integration_test.go b/go/internal/backends/protocol_anthropic_integration_test.go index 61d1b2e4..d282e3da 100644 --- a/go/internal/backends/protocol_anthropic_integration_test.go +++ b/go/internal/backends/protocol_anthropic_integration_test.go @@ -46,6 +46,14 @@ func TestMigration155ProtocolCheckAdmitsAnthropic(t *testing.T) { t.Errorf("CHECKs mentioning protocol = %d (err %v), want exactly 1", n, err) } + // The DROP/ADD of the constraint leaves the column's DEFAULT alone. + var def2 string + if err := pool.QueryRow(ctx, + `SELECT column_default FROM information_schema.columns + WHERE table_name = 'context_backends' AND column_name = 'protocol'`).Scan(&def2); err != nil || !strings.Contains(def2, "'openai'") { + t.Errorf("protocol column default = %q (err %v), want 'openai'", def2, err) + } + if _, err := pool.Exec(ctx, `INSERT INTO context_backends (name, base_url, protocol, api_key_ref, locality, num_ctx) VALUES ('anthropic-it', 'https://api.anthropic.com', 'anthropic', 'anthropic-chat', 'external', 100000)`); err != nil { diff --git a/go/internal/llm/anthropic_live_test.go b/go/internal/llm/anthropic_live_test.go index 3b096e30..a13a5185 100644 --- a/go/internal/llm/anthropic_live_test.go +++ b/go/internal/llm/anthropic_live_test.go @@ -1,6 +1,8 @@ // Live smoke of the anthropic wire against api.anthropic.com. Skipped under -// -short and without CTX_ANTHROPIC_SMOKE_KEY, so CI never spends tokens; run -// it by hand from a shell that holds the key: +// -short and without CTX_ANTHROPIC_SMOKE_KEY, so CI never spends tokens — +// note the integration job runs WITHOUT -short, so there the gate is the +// absent variable alone (nothing in this file does work before the skip). +// Run it by hand from a shell that holds the key: // // CTX_ANTHROPIC_SMOKE_KEY=sk-ant-… go test ./internal/llm/ -run Live -count=1 -v // @@ -128,12 +130,17 @@ func TestAnthropicLiveStreamToolRoundTrip(t *testing.T) { Name: "get_weather", Description: "Current weather for a city", Parameters: json.RawMessage(`{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}`), }} + // A short tool turn may come back without any thinking block (adaptive + // thinking skips trivial turns — observed live 2026-10-11), which would + // leave the replay path unexercised. Ask for visible reasoning and a + // question that needs a step of deliberation before the call. + b.ExtraBody = map[string]any{"thinking": map[string]any{"type": "adaptive", "display": "summarized"}} msgs := []ChatMsg{ - {Role: "system", Content: "You are a weather assistant. Use the get_weather tool to answer weather questions; never guess."}, - {Role: "user", Content: "What is the weather in Berlin right now? Use the tool."}, + {Role: "system", Content: "You are a weather assistant. Use the get_weather tool to answer weather questions; never guess. Think about which city the user means before calling the tool."}, + {Role: "user", Content: "I am in the capital of Germany. Is it warm enough for a t-shirt outside right now? Use the tool."}, } var events []StreamEvent - res, err := ChatStreamBackend(ctx, b, b.Model, msgs, []ToolDef{tool}, Options{NumPredict: 1024}, 90*time.Second, collectEvents(&events)) + res, err := ChatStreamBackend(ctx, b, b.Model, msgs, []ToolDef{tool}, Options{NumPredict: 2048, Extra: map[string]any{"effort": "high"}}, 90*time.Second, collectEvents(&events)) if err != nil { t.Fatalf("first stream: %v", err) } @@ -167,9 +174,14 @@ func TestAnthropicLiveStreamToolRoundTrip(t *testing.T) { if final.CacheReadTokens == 0 { t.Logf("note: no cache read on the second iteration (prefix below the 512-token minimum is expected for this tiny prompt)") } + if len(res.ProviderBlocks) == 0 { + t.Logf("REPLAY PATH NOT EXERCISED: the first turn carried no thinking block, so the round trip above and the one below are identical requests") + return + } // The same round trip WITHOUT the thinking block is what the wire did - // before the review fix — record what the API says about it today. + // before the review fix — record what the API says about a STRIPPED + // block today (only meaningful when one was produced, see above). noThinking := append([]ChatMsg{}, msgs[:2]...) noThinking = append(noThinking, ChatMsg{Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls}, diff --git a/go/internal/llm/anthropic_stream.go b/go/internal/llm/anthropic_stream.go index 48411f2e..974e4277 100644 --- a/go/internal/llm/anthropic_stream.go +++ b/go/internal/llm/anthropic_stream.go @@ -68,9 +68,14 @@ type anthropicToolResultBlock struct { // anthropicThinkingBlock / anthropicRedactedThinkingBlock are the reasoning // blocks the stream parser re-assembles so the harness can replay them -// unchanged (the API requires the final assistant turn of a tool loop to -// start with its thinking block; the text is empty under the default -// display and the signature carries the reasoning). +// unchanged. The claude-api reference asks that an assistant turn whose +// tool_use still awaits its tool_result keep its thinking block, and the +// preserved-thinking check rejects a block that was ALTERED; the text is +// empty under the default display and the signature carries the +// reasoning. Live (claude-opus-5-5, 2026-10-11) a short tool-call turn +// came back with no thinking block at all, and the round trip was +// accepted either way — so this is the reference's rule honoured, not a +// 400 observed. type anthropicThinkingBlock struct { Type string `json:"type"` // "thinking" Thinking string `json:"thinking"` @@ -141,8 +146,8 @@ func anthropicMessages(msgs []ChatMsg) (system string, out []anthropicMessage) { case "assistant": flushOrphans() var blocks []any - // Reasoning blocks first, verbatim — the API's ordering rule for - // the final assistant turn of a tool loop. + // Reasoning blocks first, verbatim — the order the API produced + // them in (thinking precedes the tool_use it introduced). for _, raw := range m.ProviderBlocks { blocks = append(blocks, raw) } diff --git a/go/internal/llm/anthropic_stream_test.go b/go/internal/llm/anthropic_stream_test.go index 8a308d03..f2efd645 100644 --- a/go/internal/llm/anthropic_stream_test.go +++ b/go/internal/llm/anthropic_stream_test.go @@ -503,8 +503,8 @@ func TestAnthropicMessagesRepairsEmptyToolIDs(t *testing.T) { // TestAnthropicStreamThinkingBlocksReplayFirst: the fixture's thinking block // (empty text, signature — the default display) is re-assembled into // StreamResult.ProviderBlocks and placed FIRST in the replayed assistant -// turn, ahead of text and tool_use — the API's ordering rule for the final -// assistant turn of a tool loop on the Claude 5 models. +// turn, ahead of text and tool_use — the order the API produced them in, +// which the reference asks to be kept while a tool_use awaits its result. func TestAnthropicStreamThinkingBlocksReplayFirst(t *testing.T) { srv := serveFixture(t, "anthropic_tool_use.sse") defer srv.Close() From db4d7eea229e00595b600b87cf81af3804d4b1c0 Mon Sep 17 00:00:00 2001 From: Damien Moon Date: Sun, 11 Oct 2026 12:56:02 +0200 Subject: [PATCH 16/16] docs(llm): wording nits from the pre-push review of the thinking-replay commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "accepted either way" said two variants were sent when the live run sent one (no block was produced); the stale header line said thinking deltas are ignored while the parser accumulates them into the replay block; the live test's stripped-block log no longer calls a rejection "expected" (the reference only asks to keep the block). In the live test the second and the stripped third round-trip call now send the same effort ("high") as the first — an effort change invalidates the messages cache by itself and would mask the prefix question — and the second call's NumPredict rises 1024 → 2048 to match the first, headroom for the high-effort turn. Co-Authored-By: Claude Fable 5.1 --- go/internal/llm/anthropic_live_test.go | 19 +++++++++++-------- go/internal/llm/anthropic_stream.go | 10 +++++----- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/go/internal/llm/anthropic_live_test.go b/go/internal/llm/anthropic_live_test.go index a13a5185..ded6200f 100644 --- a/go/internal/llm/anthropic_live_test.go +++ b/go/internal/llm/anthropic_live_test.go @@ -7,10 +7,11 @@ // CTX_ANTHROPIC_SMOKE_KEY=sk-ant-… go test ./internal/llm/ -run Live -count=1 -v // // It proves what the fakes cannot: the request shape is accepted by the real -// endpoint, the prompt cache hits on the second call, the thinking block is -// replayed correctly through a tool round trip, effort is accepted, the -// 1-token probe's behaviour, and the 401 classification. Each probe logs the -// usage it saw. Cost: a few thousand tokens per run. +// endpoint, the prompt cache hits on the second call, a tool round trip +// completes (replaying the thinking block when the model emits one — it +// logs when it did not), effort is accepted, the 1-token probe's behaviour, +// and the 401 classification. Each probe logs the usage it saw. Cost: a few +// thousand tokens per run. package llm import ( @@ -162,7 +163,9 @@ func TestAnthropicLiveStreamToolRoundTrip(t *testing.T) { ChatMsg{Role: "tool", Content: `{"temperature_c": 21, "sky": "clear"}`, ToolCallID: call.ID}, ) events = events[:0] - final, err := ChatStreamBackend(ctx, b, b.Model, msgs, []ToolDef{tool}, Options{NumPredict: 1024}, 90*time.Second, collectEvents(&events)) + // Same effort as the first iteration: a changed effort invalidates the + // prompt cache by itself and would mask the prefix question below. + final, err := ChatStreamBackend(ctx, b, b.Model, msgs, []ToolDef{tool}, Options{NumPredict: 2048, Extra: map[string]any{"effort": "high"}}, 90*time.Second, collectEvents(&events)) if err != nil { t.Fatalf("second stream (thinking replay): %v", err) } @@ -187,10 +190,10 @@ func TestAnthropicLiveStreamToolRoundTrip(t *testing.T) { ChatMsg{Role: "assistant", Content: res.Content, ToolCalls: res.ToolCalls}, ChatMsg{Role: "tool", Content: `{"temperature_c": 21, "sky": "clear"}`, ToolCallID: call.ID}, ) - if _, err := ChatStreamBackend(ctx, b, b.Model, noThinking, []ToolDef{tool}, Options{NumPredict: 256}, 90*time.Second, nil); err != nil { - t.Logf("round trip without the thinking block: %v (expected on the Claude 5 models — this is why ProviderBlocks exist)", err) + if _, err := ChatStreamBackend(ctx, b, b.Model, noThinking, []ToolDef{tool}, Options{NumPredict: 256, Extra: map[string]any{"effort": "high"}}, 90*time.Second, nil); err != nil { + t.Logf("round trip with the produced thinking block STRIPPED: rejected — %v (the reference only asks to keep it; this records what the API does)", err) } else { - t.Logf("round trip without the thinking block: accepted by the API today") + t.Logf("round trip with the produced thinking block STRIPPED: accepted by the API today") } } diff --git a/go/internal/llm/anthropic_stream.go b/go/internal/llm/anthropic_stream.go index 974e4277..1c72d111 100644 --- a/go/internal/llm/anthropic_stream.go +++ b/go/internal/llm/anthropic_stream.go @@ -12,8 +12,8 @@ // text/thinking: nothing // - content_block_delta→ text_delta: ContentDelta; input_json_delta: // arguments fragment; thinking_delta / -// signature_delta: ignored (empty under the -// default display anyway) +// signature_delta: accumulated into the replay +// block, never surfaced to the sink // - message_delta → stop_reason (mapped, FinishReason event) and // usage.output_tokens // - message_stop → done @@ -73,9 +73,9 @@ type anthropicToolResultBlock struct { // preserved-thinking check rejects a block that was ALTERED; the text is // empty under the default display and the signature carries the // reasoning. Live (claude-opus-5-5, 2026-10-11) a short tool-call turn -// came back with no thinking block at all, and the round trip was -// accepted either way — so this is the reference's rule honoured, not a -// 400 observed. +// came back with no thinking block at all and the block-less round trip +// was accepted — so this is the reference's rule honoured, not a 400 +// observed (what the API does with a STRIPPED block was not exercised). type anthropicThinkingBlock struct { Type string `json:"type"` // "thinking" Thinking string `json:"thinking"`