fix(backfill): dedicated embed-backfill arm and a time budget for the query path

docker-compose.yml

@@ -362,8 +362,10 @@ services:
       # ── status.* (1) ──
       CTX_STATUS_CHANNEL_PROBE_INTERVAL: ${CTX_STATUS_CHANNEL_PROBE_INTERVAL:-}
 
-      # ── embed_backfill.* (4) ──
+      # ── embed_backfill.* (6) ──
       CTX_EMBED_BACKFILL_SYNC_CAP: ${CTX_EMBED_BACKFILL_SYNC_CAP:-}
+      CTX_EMBED_BACKFILL_SYNC_BUDGET: ${CTX_EMBED_BACKFILL_SYNC_BUDGET:-}
+      CTX_EMBED_BACKFILL_INTERVAL: ${CTX_EMBED_BACKFILL_INTERVAL:-}
       CTX_EMBED_BACKFILL_MAX_TOKENS: ${CTX_EMBED_BACKFILL_MAX_TOKENS:-}
       CTX_EMBED_BACKFILL_BACKOFF_BASE: ${CTX_EMBED_BACKFILL_BACKOFF_BASE:-}
       CTX_EMBED_BACKFILL_BACKOFF_CAP: ${CTX_EMBED_BACKFILL_BACKOFF_CAP:-}

docs/operations.md

@@ -477,6 +477,8 @@ The `llama-embed` sidecar (`docker-compose.override.yml`, Qwen3-Embedding-8B, `-
 
 **Backfill failure memo (migration 113, Achse 04 W04-2).** Both backfill paths — the scheduler arm (`backfillOneEmbedding`) and the query-path pre-search loop (`backfillPending`) — used to retry a NULL-embedding block **indefinitely** if it exceeded the embed window: no truncate, no skip, nothing written to the DB on failure, so the next pick landed on the exact same block (observed 2026-07-10: a 13k-char block at 4704 tokens against a 4096 window looped every ~20 s and blocked ~265 pending blocks behind it). This is closed structurally, not operationally: every failed embed attempt now upserts a row into `context_embed_failures` (`block_id`, `attempts`, `last_error` — normalized, ≤500 chars, never raw wire/embed-text content —, `last_class`, `next_attempt_at`), and both pick queries exclude a block whose `next_attempt_at` is still in the future. A non-oversize failure (backend down, transient wire error) gets an exponential backoff (`embed_backfill.backoff_base` × 2^(attempts-1), capped at `embed_backfill.backoff_cap`; defaults 1min/24h; the base must be > 0 — V21 rejects 0, which would mean "retry immediately" and a tight loop against a failing backend, issue #38) — the next PEEK reaches the next-oldest block instead of re-picking the same one. An oversize block — classified either by a conservative pre-wire token estimate (`len(text)/4` against `embed_backfill.max_tokens`, default 24000, chosen to sit under the live kv-unified pool's 32k minus interactive headroom — see the tuning section above) or by an HTTP 400 whose body names `exceed_context_size` — gets `next_attempt_at = 'infinity'` instead: permanently parked, never a blind 24h-in-slow-motion retry, but never silently dropped either (the memo row stays visible in `context_embed_failures`). Known limitation: `ClearEmbedding` does NOT currently clear the block's failure memo, so a content edit that shrinks a previously oversize-parked block below the token threshold does **not** reactivate it on its own — the `infinity` exclusion persists until the memo row is cleaned up (a janitor for this is not yet built; see the Achse-04 backlog). The query-path loop additionally carries a synchronous cap (`embed_backfill.sync_cap`, default 4): before this wave a large rest-transient (e.g. after a bulk ingest that outran the scheduler arm) made the *first* interactive query pay the entire nachzug inline; now it embeds at most `sync_cap` blocks per request and leaves the rest to the scheduler arm. Keeping blocks compact (~1–1.5k chars per the store guidance) or sizing the embed window above your largest block is still the actual fix for an oversize block's content — the memo mechanism only stops it from starving everything behind it.
 
+**The dedicated backfill arm and the query-path time budget (tracking issue: force-push.me/damienmoon/ctx/issues/1).** Two gaps in the above showed up on a CPU embed backend (Qwen3-Embedding-8B at the documented ~23 tok/s) with a ~10k-token block. First, the scheduler arm ran only at the top of the dream loop, so with `dream.enabled=false` — the default — nothing backfilled pending embeddings in the background at all; the query path was the only path. `runEmbedBackfillLoop` is now its own goroutine, started regardless of Dream: a successful pick loops at once, an empty pick waits `embed_backfill.interval` (default 15 s), `0` turns the arm off (polled every 15 s for a hot re-enable). The dream loop keeps its inline call; the tx-wrapped `FOR UPDATE SKIP LOCKED` pick makes two pickers safe. Second, `sync_cap` bounds how *many* blocks a query embeds inline, not how *long*: one block that needs 7 minutes consumed the caller's whole 120 s deadline, the question's own embed was never admitted, and the query answered 500 instead of degrading. `embed_backfill.sync_budget` (seconds, default 30, `0` = unbounded) now bounds the whole pre-search loop per request; each wire call runs under what is left of it. A block that overruns the budget — or dies with the request context — is memoized with the class `caller_timeout`: the query path never picks it again (`EmbedFailureQueryPathExcludedPredicate`), the background arm picks it at once (`next_attempt_at = now()`), and a later wire failure there reclassifies it as `wire` with the ordinary backoff. The block stays FTS-visible meanwhile. Previously such a failure was memoized as `wire` and re-burned the deadline of every query that landed after the backoff lapsed.
+
 **Backfill skips retrieval-excluded types (2026-07-25).** Both backfill paths now also skip a block whose *type* carries retrieval policy `excluded` in the block-type registry (shipped builtins: `checkpoint`, `system-meta`). Such a vector has no consumer at all — every ranked path filters against the registry's visible-type allowlist, so the embedding was written once and then read by nobody. The trigger was a live incident on 2026-07-25 with two independent arms: 33 Hermes "Compaction source" parts (`type=checkpoint`, ~36.5 kB each) saturated the CPU embed backend for hours, and the digest rebuild rewrites `topic-map-private` (`type=system-meta`, 73.7 kB ≈ ~32k real tokens) on **every** boot, which clears its embedding and bought ~60 min of CPU embed for a dead vector — recurring, forever. The filter is policy-driven, not a hardcoded type list (same mechanic as `test.sh` T19), and deliberately conservative in the fail-safe direction: only `_global`-`excluded` skips; `damped` and `aggregate-to-parent` stay embeddable, a block whose type has no registry row at all stays embeddable, and a tenant scope that overrides an excluded type to a visible policy re-opens that type for the backfill (the pick has no reader identity, so the reader-aware overlay rule is mirrored as the union over all possible readers). Two operational consequences: blocks of an excluded type keep `embedding IS NULL` forever — that is the intended end state, not a stuck queue — and the `/status` `embed_backlog` metric still counts them, so a corpus with such blocks shows a permanent backlog floor (live at the time of writing: 24 `checkpoint` + 1 `system-meta`). Existing `infinity` failure memos for such blocks become moot: the predicate no longer even sees the rows. Flipping a type's policy from `excluded` to a visible one makes its blocks embeddable again on the next cycle without any manual step.
 
 **Re-embed migration worker (Achse 04 W04-4).** With an active migration in `running` or `verifying`, a dedicated scheduler arm (`migrateOneEmbedding`, BACKGROUND class, defers to interactive demand) walks all blocks that carry an old-space vector (`embedding IS NOT NULL AND embedding_next IS NULL AND NOT is_archived`) oldest-first from a persistent cursor, embeds them via the model-map key `embed_next` on the regular embed chain, and writes `embedding_next`/`embed_model_next` — the serving pair stays byte-identical until the cutover. Operator knobs: `embed_migration.batch_per_cycle` (default 8, `<=0` disables the arm), `embed_migration.max_tokens` (default 24000, pre-wire oversize gate; `0` disables the estimate — the wire classification of HTTP 400 `exceed_context_size` stays), `embed_migration.backoff_base`/`backoff_cap` (defaults 1min/24h; the base must be > 0 — V21, same tight-loop reasoning as the backfill sibling) — all hot, global-only. Failure memos are migration-scoped rows in `context_embed_failures` (independent from the regular backfill's memos): transient wire errors back off exponentially, `oversize` and `sensitivity_ineligible` (no backend trusted for the block's sensitivity) park at `infinity` — visible in the memo table, surfaced in status/state.sh with W04-7. **If a migration sits in `paused` with `last_error = "model guard: resolved <X>, expected <Y>"`**: the target backend's model_map lost (or never had) the `embed_next` key resolving to the migration's `to_model` — fix the backend row, then resume; the guard re-checks every cycle and exists because a missing key silently falls back to the OLD model (relabeling old vectors as migrated). The arm maintains a runtime partial index `idx_embedding_next_pending` (created `CONCURRENTLY` when a migration is active, INVALID leftovers from a crashed build are dropped and rebuilt automatically, dropped again once the migration is terminal) — seeing it in the catalog is normal only while a migration exists. Measured backend behavior (2026-07-24, CPU llama.cpp, kv-unified): a parallel embed that overflows the shared pool WAITS (no error path), and even below the pool limit a small interactive embed queues behind a large prefill — at ~20 tok/s CPU prefill a 24k-token block holds a slot for ~19 min, which is what `max_tokens` actually bounds.

go/internal/config/config.go

@@ -1592,6 +1592,29 @@ type EmbedBackfillConfig struct {
 	// unit test that never sets this field disables the cap rather than
 	// silently blocking every backfillPending call.
 	SyncCap int `key:"embed_backfill.sync_cap" env:"CTX_EMBED_BACKFILL_SYNC_CAP" default:"4" mut:"hot" tenancy:"global-only"`
+	// SyncBudget bounds Pfad A in SECONDS, next to SyncCap's count: the
+	// whole pre-search loop of one request runs inside this wall-clock
+	// budget, and each block's embed call runs under what is left of it.
+	// SyncCap alone could not stop one slow block from eating the caller's
+	// entire deadline (the tracking issue, force-push.me/damienmoon/ctx/issues/1: a ~10k-token block against a
+	// CPU embed backend at ~23 tok/s needs ~7 min, the CLI client's deadline
+	// is 120 s — the question's own embed was never admitted and the query
+	// answered 500 instead of degrading). A block that overruns the budget
+	// is memoized as caller_timeout (store.EmbedFailureCallerTimeout) and
+	// left to the background arm; the question embed keeps the rest of the
+	// deadline. Default 30 s (a quarter of the CLI deadline). 0 = unbounded,
+	// the pre-budget behavior (explicit opt-out, SyncCap convention).
+	SyncBudget time.Duration `key:"embed_backfill.sync_budget" env:"CTX_EMBED_BACKFILL_SYNC_BUDGET" default:"30" mut:"hot" tenancy:"global-only"`
+	// Interval is the idle wait in seconds of the dedicated background
+	// backfill arm (runEmbedBackfillLoop) between two empty picks. Until
+	// this key the only background backfill was the call at the top of the
+	// dream loop, so with dream.enabled=false (the default) NOTHING embedded
+	// pending blocks in the background and Pfad A was the only path (issue
+	// force-push.me/damienmoon/ctx/issues/1). The arm runs regardless of Dream; a successful
+	// pick loops at once, an empty pick waits this long. 0 = arm off
+	// (polled every 15 s for a hot re-enable, contract.recheck_interval
+	// convention).
+	Interval time.Duration `key:"embed_backfill.interval" env:"CTX_EMBED_BACKFILL_INTERVAL" default:"15" mut:"hot" tenancy:"global-only"`
 	// MaxTokens is the pre-wire Oversize-Gate estimate threshold
 	// (design/04 §4.4): len(embedText)/4 above this skips the block WITHOUT
 	// a wire call (last_class='oversize', next_attempt_at='infinity' —

go/internal/config/descriptions.go

@@ -175,6 +175,8 @@ var keyDescriptions = map[string]string{ //nolint:gosec // G101-Fehlalarm: Schl
 	"contract.recheck_interval":                      "Seconds between periodic schema-drift re-checks; 0 runs no checks while the ticker keeps polling so a later hot enable takes effect",
 	"status.channel_probe_interval":                  "Seconds between latency probes of the four retrieval channels (semantic, fts_de, fts_en, trigram); 0 (default) disables the probe",
 	"embed_backfill.sync_cap":                        "Maximum pending blocks a query embeds inline before its search runs; 0 = uncapped, the cap bounds first-query latency after a backlog",
+	"embed_backfill.sync_budget":                     "Seconds a query may spend embedding pending blocks before its own search; a block that overruns it is memoized as caller_timeout and left to the background arm, 0 = unbounded",
+	"embed_backfill.interval":                        "Seconds the background embed-backfill arm waits between two empty picks; it runs regardless of Dream, 0 = arm off (polled for a hot re-enable)",
 	"embed_backfill.max_tokens":                      "Estimated-token threshold (length/4) above which a block is permanently parked as oversize without a wire call; 0 disables the pre-check",
 	"embed_backfill.backoff_base":                    "Base delay in seconds of the exponential retry curve (base * 2^(attempts-1)) for transient embed failures. Must be > 0 (V21) — 0 would mean retry immediately, a tight loop against a failing embed backend",
 	"embed_backfill.backoff_cap":                     "Upper bound in seconds on the embed-failure retry delay; a persistently failing block retries at most this often",

go/internal/config/validate_test.go

@@ -393,7 +393,7 @@ func TestV17bIsAdvisoryOnly(t *testing.T) {
 // changes TYPE (seconds → int, or a struct field that stops being a
 // time.Duration) would leave the walk without anyone noticing. Raise this
 // number in the same commit that adds a duration key.
-const wantDurationKeys = 45
+const wantDurationKeys = 47 // +2 on 2026-10-09: embed_backfill.sync_budget and embed_backfill.interval, both V17-covered
 
 // TestValidateRejectsNegativeOnEveryDurationKey is the registry-wide form of
 // V17 (issue #29): before it, exactly two of the seconds keys had a sign check

go/internal/events/backfill_arm_integration_test.go

@@ -0,0 +1,124 @@
+//go:build integration
+
+// Rot-Gate for the tracking issue, force-push.me/damienmoon/ctx/issues/1, scheduler half: the embed-backfill
+// arm (backfillOneEmbedding) was called ONLY at the top of runDreamLoop, so
+// with dream.enabled=false — the registry default — no goroutine ever
+// embedded a pending block in the background. Live: a block stayed pending
+// for two hours while the scheduler logged guard and digest ticks only; the
+// query path was the sole embedder, inside its caller's deadline.
+//
+// Rot is the premise, stated against the production wiring: a scheduler
+// with DreamEnabled=false starts no dream worker, and nothing else in Run
+// called backfillOneEmbedding before this wave. Grün is the dedicated arm
+// (runEmbedBackfillLoop): started regardless of Dream, it embeds the pending
+// block within a few intervals, and a caller_timeout memo (the query path's
+// hand-off) does not hold it back.
+//
+// Run: `go test -tags=integration ./internal/events/ -run TestEmbedBackfillArm -count=1 -v`.
+package events
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/GottZ/ctx/internal/backends"
+	"github.com/GottZ/ctx/internal/config"
+	"github.com/GottZ/ctx/internal/dispatch"
+	"github.com/GottZ/ctx/internal/store"
+	"github.com/GottZ/ctx/internal/testdb"
+)
+
+func TestEmbedBackfillArm_RunsWithoutDream_Integration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("integration test")
+	}
+	pool := testdb.SetupTestDB(t)
+	ctx := context.Background()
+
+	srv := headOfLineEmbedServer(t)
+	bpool := backends.NewPool(nil, nil)
+	bpool.SeedSnapshotForTest([]backends.Backend{embedPoolRow("embed-a", srv.URL, 100)})
+	d := dispatch.New(nil, dispatch.DefaultSettings()) // empty policy: pass-through
+	t.Cleanup(d.Close)
+
+	cfg := &config.Config{EmbedBackfill: config.EmbedBackfillConfig{
+		Interval:    100 * time.Millisecond,
+		MaxTokens:   0,
+		BackoffBase: time.Hour,
+		BackoffCap:  24 * time.Hour,
+	}}
+	// DreamEnabled=false: the RED premise — before this wave, Run started no
+	// goroutine that could reach backfillOneEmbedding in this configuration.
+	s := NewScheduler(pool, config.NewStore(cfg), bpool, StartupConfig{DreamEnabled: false})
+	s.SetDispatcher(d)
+
+	waitEmbedded := func(t *testing.T, within time.Duration) bool {
+		t.Helper()
+		deadline := time.Now().Add(within)
+		for time.Now().Before(deadline) {
+			if pendingCount(t, pool) == 0 {
+				return true
+			}
+			time.Sleep(25 * time.Millisecond)
+		}
+		return false
+	}
+
+	t.Run("arm_embeds_a_pending_block_with_dream_off", func(t *testing.T) {
+		seedPendingBlock(t, pool, "arm-pending-normal", time.Hour)
+		defer clearBlocks(t, pool)
+
+		armCtx, stop := context.WithCancel(ctx)
+		defer stop()
+		go s.runEmbedBackfillLoop(armCtx)
+
+		if !waitEmbedded(t, 5*time.Second) {
+			t.Fatalf("block still pending after 5 s: the dedicated arm did not embed it (pending=%d)", pendingCount(t, pool))
+		}
+	})
+
+	t.Run("arm_picks_a_caller_timeout_block_at_once", func(t *testing.T) {
+		seedPendingBlock(t, pool, "arm-handed-off", time.Hour)
+		defer clearBlocks(t, pool)
+
+		// The query path's hand-off: a caller_timeout memo, due now.
+		var blockID string
+		if err := pool.QueryRow(ctx, `SELECT id FROM context_blocks WHERE title = 'arm-handed-off'`).Scan(&blockID); err != nil {
+			t.Fatalf("read block id: %v", err)
+		}
+		if err := store.RecordEmbedFailure(ctx, pool, blockID, store.EmbedFailureCallerTimeout,
+			store.NormalizeEmbedError(store.EmbedFailureCallerTimeout, "context deadline exceeded"),
+			cfg.EmbedBackfill.BackoffBase, cfg.EmbedBackfill.BackoffCap); err != nil {
+			t.Fatalf("record caller_timeout memo: %v", err)
+		}
+		t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM context_embed_failures WHERE block_id = $1`, blockID) })
+
+		armCtx, stop := context.WithCancel(ctx)
+		defer stop()
+		go s.runEmbedBackfillLoop(armCtx)
+
+		if !waitEmbedded(t, 5*time.Second) {
+			t.Fatalf("caller_timeout block still pending after 5 s: the arm must pick it at once, not after a backoff")
+		}
+	})
+
+	t.Run("interval_zero_turns_the_arm_off", func(t *testing.T) {
+		seedPendingBlock(t, pool, "arm-off-pending", time.Hour)
+		defer clearBlocks(t, pool)
+
+		off := *cfg
+		off.EmbedBackfill.Interval = 0
+		sOff := NewScheduler(pool, config.NewStore(&off), bpool, StartupConfig{DreamEnabled: false})
+		sOff.SetDispatcher(d)
+
+		armCtx, stop := context.WithCancel(ctx)
+		defer stop()
+		go sOff.runEmbedBackfillLoop(armCtx)
+
+		time.Sleep(600 * time.Millisecond)
+		if got := pendingCount(t, pool); got != 1 {
+			t.Fatalf("pending = %d, want 1 (Interval=0 is the explicit off switch)", got)
+		}
+	})
+}