diff --git a/apps/loopover-ui/src/lib/selfhost-env-reference.ts b/apps/loopover-ui/src/lib/selfhost-env-reference.ts index e97b8aea4..a1ee96f9e 100644 --- a/apps/loopover-ui/src/lib/selfhost-env-reference.ts +++ b/apps/loopover-ui/src/lib/selfhost-env-reference.ts @@ -119,7 +119,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "BROWSER_WS_ENDPOINT", - firstReference: "src/selfhost/stubs/puppeteer.ts", + firstReference: "src/selfhost/health.ts", }, { name: "CLAUDE_AI_EFFORT", @@ -333,6 +333,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "MAINTENANCE_ADMISSION_MAX_HOST_LOAD", firstReference: "src/selfhost/maintenance-admission.ts", }, + { + name: "MAINTENANCE_ADMISSION_MAX_HOST_MEMORY", + firstReference: "src/selfhost/maintenance-admission.ts", + }, { name: "MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS", firstReference: "src/selfhost/maintenance-admission.ts", @@ -698,7 +702,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `ANTHROPIC_AI_MODEL` | `src/selfhost/ai.ts` |", "| `ANTHROPIC_API_KEY` | `src/selfhost/ai-config.ts` |", "| `BACKUP_ACKNOWLEDGED` | `src/server.ts` |", - "| `BROWSER_WS_ENDPOINT` | `src/selfhost/stubs/puppeteer.ts` |", + "| `BROWSER_WS_ENDPOINT` | `src/selfhost/health.ts` |", "| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts` |", "| `CLAUDE_AI_FIRST_OUTPUT_TIMEOUT_MS` | `src/selfhost/ai.ts` |", "| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts` |", @@ -752,6 +756,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `MAINTENANCE_ADMISSION_MAX_BACKLOG_CONVERGENCE_PENDING` | `src/selfhost/maintenance-admission.ts` |", "| `MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS` | `src/selfhost/maintenance-admission.ts` |", "| `MAINTENANCE_ADMISSION_MAX_HOST_LOAD` | `src/selfhost/maintenance-admission.ts` |", + "| `MAINTENANCE_ADMISSION_MAX_HOST_MEMORY` | `src/selfhost/maintenance-admission.ts` |", "| `MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS` | `src/selfhost/maintenance-admission.ts` |", "| `MAINTENANCE_ADMISSION_MAX_LIVE_PENDING` | `src/selfhost/maintenance-admission.ts` |", "| `MAINTENANCE_ADMISSION_MAX_PENDING` | `src/selfhost/maintenance-admission.ts` |", diff --git a/src/selfhost/blob-store.ts b/src/selfhost/blob-store.ts index 479b1018a..9897028c4 100644 --- a/src/selfhost/blob-store.ts +++ b/src/selfhost/blob-store.ts @@ -5,7 +5,8 @@ // surface those two paths use; every other R2Bucket method is unused on self-host. Node-only (fs import never // reaches the Worker bundle — wired in server.ts behind REVIEW_AUDIT_DIR). MODULAR + off by default: unset // REVIEW_AUDIT_DIR ⇒ no REVIEW_AUDIT binding ⇒ captures degrade to on-demand exactly as before. -import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; import { dirname, resolve, sep } from "node:path"; /** Build a filesystem-backed REVIEW_AUDIT store rooted at `baseDir`. Keys are app-generated @@ -48,7 +49,27 @@ export function createFsBlobStore(baseDir: string): R2Bucket { async put(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | Blob | null): Promise { const target = pathFor(key); await mkdir(dirname(target), { recursive: true }); - await writeFile(target, Buffer.from(await new Response(value ?? "").arrayBuffer())); + // #9487: write to a unique temp path, then rename. Keys here are INPUT-HASH-ADDRESSED + // (`loopover/shots/.png`), so a half-written file from a mid-write kill is never retried or + // overwritten -- the next lookup for that same input hits it, finds a file, and serves a truncated PNG + // as a permanently "valid" cache entry. rename(2) is atomic within a filesystem, so a reader sees + // either no file or the complete one, never a partial. Same tmp+rename the config writer already does + // (private-config.ts's atomicWriteWithBackup) -- and the temp name carries a UUID so two concurrent + // puts of the same key cannot clobber each other's in-progress file. + const tmpTarget = `${target}.tmp-${randomUUID()}`; + try { + await writeFile(tmpTarget, Buffer.from(await new Response(value ?? "").arrayBuffer())); + await rename(tmpTarget, target); + } catch (error) { + // Never leave the temp file behind on a failed write -- otherwise a full disk or a mid-write crash + // accretes orphans in the same directory the real objects live in. Best-effort: the original error is + // what the caller needs, not a cleanup failure. + /* v8 ignore next -- the cleanup's own failure arm: `force: true` already swallows ENOENT, so this only + fires for something like an unwritable directory, in which case the ORIGINAL write error is what the + caller needs. Unreachable without mocking fs, and mocking it here would test the mock. */ + await rm(tmpTarget, { force: true }).catch(() => undefined); + throw error; + } return { key } as unknown as R2Object; }, /** Remove a stored object. A missing file is not an error (matches R2's own delete-is-idempotent diff --git a/src/selfhost/health.ts b/src/selfhost/health.ts index 2036e5a72..6c4160090 100644 --- a/src/selfhost/health.ts +++ b/src/selfhost/health.ts @@ -143,6 +143,75 @@ export function codexAuthReadinessProbe( }; } +/** + * #9487/#9464: readiness probe for the browserless endpoint backing visual capture. + * + * Every other dependency the review path needs has a probe (Redis, Qdrant, the GitHub App, codex); + * `BROWSER_WS_ENDPOINT` had none, so a browserless outage was invisible in `/ready` AND in Prometheus while it + * silently changed gate OUTCOMES — the screenshot-table gate reads absent evidence as a close signal. #9464 + * stopped that from closing PRs; this makes the outage itself visible instead of inferred after the fact. + * + * Only registered when the endpoint is configured — an instance with visual review switched off is not + * degraded for lacking a browser, so an unconditional probe would make `/ready` red for every deployment that + * never wanted the feature. + * + * Probes the endpoint's HTTP sibling rather than opening a websocket + launching Chromium: a readiness check + * must be cheap enough to run on every poll, and a real launch costs a browser process. browserless answers + * `/json/version` over plain HTTP on the same host/port as its `ws://` endpoint, which is exactly the + * "is the service accepting connections" signal wanted here. + * + * Result is cached like the codex probe (same `cacheMs` shape, same single-flight guard) so a poll loop can't + * turn readiness into its own load source against an already-struggling backend. + */ +export function browserEndpointReadinessProbe( + env: Record, + fetchImpl: (url: string) => Promise<{ ok: boolean }>, + cacheMs = 30_000, +): ReadinessProbe | null { + const endpoint = (env.BROWSER_WS_ENDPOINT ?? "").trim(); + if (!endpoint) return null; + const versionUrl = browserVersionUrl(endpoint); + // A configured-but-unparseable endpoint is a real misconfiguration, and one the ordinary capture path would + // only surface as a per-shot render failure. Fail readiness closed rather than skipping the probe. + if (versionUrl === null) return { name: "browser_endpoint", check: () => Promise.resolve(false) }; + let cached: boolean | undefined; + let cachedUntil = 0; + let inFlight: Promise | undefined; + return { + name: "browser_endpoint", + check: () => { + const now = Date.now(); + if (cached !== undefined && now < cachedUntil) return Promise.resolve(cached); + if (inFlight) return inFlight; + inFlight = fetchImpl(versionUrl) + .then((response) => response.ok) + .catch(() => false) + .then((ok) => { + cached = ok; + cachedUntil = Date.now() + cacheMs; + return ok; + }) + .finally(() => { + inFlight = undefined; + }); + return inFlight; + }, + }; +} + +/** The `http(s):///json/version` sibling of a `ws(s)://` browserless endpoint, or null when the + * configured value is not a parseable ws/wss URL. Query strings (browserless carries `?token=`) are dropped: + * the version endpoint needs no auth and the token must never end up in a probe URL that could be logged. */ +function browserVersionUrl(endpoint: string): string | null { + try { + const url = new URL(endpoint); + if (url.protocol !== "ws:" && url.protocol !== "wss:") return null; + return `${url.protocol === "wss:" ? "https:" : "http:"}//${url.host}/json/version`; + } catch { + return null; + } +} + /** Boot-time DATA-SAFETY advisory. A single SQLite file with no acknowledged backup is a data-loss SPOF — yet * `/ready` would still answer 200, so an operator can run with zero durability believing they're healthy. Returns * the warning to log at boot (or null on Postgres, or once the operator sets `BACKUP_ACKNOWLEDGED=true` after diff --git a/src/selfhost/host-pressure.ts b/src/selfhost/host-pressure.ts index 7e8c0af10..17a1584f0 100644 --- a/src/selfhost/host-pressure.ts +++ b/src/selfhost/host-pressure.ts @@ -2,7 +2,7 @@ // `node:os`'s loadavg() has no meaningful signal on Cloudflare Workers -- this module is imported ONLY by the // self-host Node queue backends (sqlite-queue.ts / pg-queue.ts), never by src/index.ts's Worker bundle, so a // static `node:os` import here is safe (mirrors the existing `hostname` import in selfhost/posthog.ts). -import { cpus, loadavg } from "node:os"; +import { cpus, freemem, loadavg, totalmem } from "node:os"; /** The 1-minute load average normalized per logical core, so the SAME threshold means the same thing on a * 4-vCPU box as a 32-vCPU box. Best-effort and fail-open: any error, or a reading that can't possibly be a @@ -20,3 +20,37 @@ export function hostLoadAvg1PerCore(): number | null { return null; } } + +/** + * #9487: the fraction of host memory currently IN USE (0..1), or `null` when unavailable. + * + * Host pressure watched CPU only, but on the deployment this was found on (a GPU box running Ollama at + * ~9.9 GiB alongside browserless at ~1.5 GiB) memory is the realistic killer: nothing observed it, nothing + * shed load for it, and the OOM killer made the decision instead — which takes the whole container, losing + * every in-flight job, rather than deferring one maintenance job. + * + * Same fail-open contract as {@link hostLoadAvg1PerCore}: any error, or a reading that cannot be a real + * ratio, yields `null` ("signal unavailable"), never a misleading 0. A caller must treat `null` as "skip this + * check". + * + * HONEST LIMIT, shared with the load signal above: `node:os` reports the HOST's memory, so under a container + * memory limit (cgroup) this understates pressure — the container can be at its own ceiling while the host + * looks idle. Reading `/sys/fs/cgroup/memory.current` would fix that and is deliberately not done here: it is + * Linux- and cgroup-v2-specific, and this module is a best-effort *hint* for admission, not an accounting + * boundary. The same caveat already applies to loadavg-over-container-cores. + */ +export function hostMemoryUsedFraction(): number | null { + try { + const total = totalmem(); + const free = freemem(); + if (!Number.isFinite(total) || total <= 0) return null; + if (!Number.isFinite(free) || free < 0) return null; + const used = (total - free) / total; + // A free reading above total would put this outside 0..1 -- treat any impossible ratio as unavailable + // rather than clamping, so a broken platform reading can never masquerade as "no pressure". + if (used < 0 || used > 1) return null; + return used; + } catch { + return null; + } +} diff --git a/src/selfhost/load-file-secrets.ts b/src/selfhost/load-file-secrets.ts index 740bc6bd4..799da1cf1 100644 --- a/src/selfhost/load-file-secrets.ts +++ b/src/selfhost/load-file-secrets.ts @@ -27,8 +27,9 @@ export function loadFileSecrets( const target = key.slice(0, -"_FILE".length); if (env[target]) continue; // an explicit value wins const path = env[key] as string; + let value: string; try { - env[target] = readFile(path).trim(); + value = readFile(path).trim(); } catch (error) { console.error( JSON.stringify({ @@ -43,5 +44,26 @@ export function loadFileSecrets( }`, ); } + // #9487: an EMPTY (zero-byte or whitespace-only) secret file is as fatal as a missing one. Setting + // `env[target] = ""` looks like a successful load, but every downstream `nonBlank()` reads "" as + // UNCONFIGURED and preflight.ts deliberately skips absent values -- so a truncated + // GITHUB_WEBHOOK_SECRET file booted an instance that silently rejected every webhook. Directly adjacent + // to the known secret-rotation footgun on edge-nl-01, where a file is rewritten in place: the window in + // which it is momentarily empty is exactly when a container restart reads it. + // + // Checked OUTSIDE the try above on purpose: throwing inside it would be caught by that catch and + // re-reported as "unreadable", collapsing two genuinely different operator problems (a bad path/permission + // vs a truncated write) into one misleading message and the wrong log event. + if (value === "") { + console.error( + JSON.stringify({ + level: "error", + event: "selfhost_secret_file_empty", + var: key, + }), + ); + throw new Error(`Secret file for ${key} (${path}) is empty; an empty secret silently reads as unconfigured downstream. Write the value, or unset ${key}.`); + } + env[target] = value; } } diff --git a/src/selfhost/maintenance-admission.ts b/src/selfhost/maintenance-admission.ts index 2950c6313..a525be480 100644 --- a/src/selfhost/maintenance-admission.ts +++ b/src/selfhost/maintenance-admission.ts @@ -108,6 +108,12 @@ export interface MaintenancePressureSignals { oldestMaintenancePendingAgeMs: number | null; /** Null when unavailable (see host-pressure.ts) -- a caller must treat null as "skip this check". */ hostLoadAvg1PerCore: number | null; + /** #9487: fraction of host memory in use (0..1), or null when unavailable -- same skip-on-null contract as + * hostLoadAvg1PerCore above. Pressure watched CPU only, but on a box also running Ollama (~9.9 GiB) and + * browserless (~1.5 GiB) memory is the realistic killer, and nothing observed it: the OOM killer decided + * instead, taking the whole container and every in-flight job with it rather than deferring one + * maintenance job. */ + hostMemoryUsedFraction: number | null; /** #selfhost-backlog-convergence: pending+processing count of `agent-regate-pr` jobs tagged * `foreground_lane='backlog'` (queue-fairness.ts) -- the backlog-convergence sweeper's own output, DISTINCT * from `livePendingCount` (which is priority-gated, not lane-gated, and includes fresh webhook/foreground @@ -128,6 +134,8 @@ export interface MaintenanceAdmissionConfig { maxLiveJobAgeMs: number; maxMaintenancePendingCount: number; maxHostLoadAvg1PerCore: number; + /** #9487: defer maintenance above this fraction of host memory in use. */ + maxHostMemoryUsedFraction: number; maxBacklogConvergencePendingCount: number; deferMs: number; maxDeferAgeMs: number; @@ -146,6 +154,10 @@ const DEFAULT_MAX_LIVE_PENDING_COUNT = 5; const DEFAULT_MAX_LIVE_JOB_AGE_MS = 2 * 60_000; const DEFAULT_MAX_MAINTENANCE_PENDING_COUNT = 15; const DEFAULT_MAX_HOST_LOAD_AVG1_PER_CORE = 1.5; +// #9487: 0.92 leaves genuine headroom before the OOM killer without deferring on ordinary steady-state usage +// -- a box running a resident model server sits legitimately high (page cache plus a multi-GiB model), so a +// tighter default would defer maintenance permanently on exactly the deployment this was found on. +const DEFAULT_MAX_HOST_MEMORY_USED_FRACTION = 0.92; // Deliberately more permissive than maxLivePendingCount (5): a real incident's backlog-convergence sweep can // legitimately queue several PRs across several repos at once (BACKLOG_CONVERGENCE_SWEEP_MAX_PRS=5 per repo per // sweep, selfhost/backlog-convergence.ts) without that alone meaning maintenance must fully yield -- only a @@ -200,6 +212,10 @@ export function resolveMaintenanceAdmissionConfig(): MaintenanceAdmissionConfig "MAINTENANCE_ADMISSION_MAX_HOST_LOAD", DEFAULT_MAX_HOST_LOAD_AVG1_PER_CORE, ), + maxHostMemoryUsedFraction: parsePositiveFloatEnv( + "MAINTENANCE_ADMISSION_MAX_HOST_MEMORY", + DEFAULT_MAX_HOST_MEMORY_USED_FRACTION, + ), maxBacklogConvergencePendingCount: parsePositiveIntEnv("MAINTENANCE_ADMISSION_MAX_BACKLOG_CONVERGENCE_PENDING", { min: 0, fallback: DEFAULT_MAX_BACKLOG_CONVERGENCE_PENDING_COUNT, @@ -225,6 +241,7 @@ export type MaintenanceAdmissionReason = | "maintenance_pending_high" | "maintenance_pending_high_drain" | "host_load_high" + | "host_memory_high" | "pressure_clear"; export interface MaintenanceAdmissionDecision { @@ -268,6 +285,9 @@ export function evaluateMaintenanceAdmission( } const hostLoadHigh = signals.hostLoadAvg1PerCore !== null && signals.hostLoadAvg1PerCore > config.maxHostLoadAvg1PerCore; + // #9487: memory sits beside CPU as a peer pressure dimension, with the identical null-means-skip contract. + const hostMemoryHigh = + signals.hostMemoryUsedFraction !== null && signals.hostMemoryUsedFraction > config.maxHostMemoryUsedFraction; if (signals.maintenancePendingCount > config.maxMaintenancePendingCount) { if (nowMs - pendingSinceMs >= config.maintenanceDrainAgeMs) { // Host load is re-checked HERE, gating the drain escape specifically: draining more maintenance work onto @@ -275,11 +295,15 @@ export function evaluateMaintenanceAdmission( // drain age yet is denied `maintenance_pending_high` regardless of host load (unchanged from before this // escape existed) -- this check only ever changes the outcome for a job the drain would otherwise admit. if (hostLoadHigh) return { admit: false, reason: "host_load_high" }; + // Memory gates the drain escape for the same reason load does: draining more work onto a box that is + // already near its memory ceiling is what invites the OOM kill this signal exists to avoid. + if (hostMemoryHigh) return { admit: false, reason: "host_memory_high" }; return { admit: true, reason: "maintenance_pending_high_drain" }; } return { admit: false, reason: "maintenance_pending_high" }; } if (hostLoadHigh) return { admit: false, reason: "host_load_high" }; + if (hostMemoryHigh) return { admit: false, reason: "host_memory_high" }; return { admit: true, reason: "pressure_clear" }; } diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index b4b184b5b..4d7cc99d1 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -57,7 +57,7 @@ export const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["loopover_backup_acknowledged", { help: "1 when SQLite backup is acknowledged or Postgres is in use; 0 when the boot backup advisory would fire.", type: "gauge" }], ["loopover_config_dir_empty_acknowledged", { help: "1 when LOOPOVER_REPO_CONFIG_DIR is unset, has entries, or is acknowledged; 0 when it's configured but the mounted directory is empty.", type: "gauge" }], ["loopover_http_requests_total", { help: "HTTP app requests by response status class.", type: "counter" }], - ["loopover_http_request_duration_seconds", { help: "HTTP app request duration in seconds.", type: "histogram" }], + ["loopover_http_request_duration_seconds", { help: "HTTP app request duration in seconds, labelled by bounded route group (see httpRouteGroup).", type: "histogram" }], ["loopover_visual_capture_total", { help: "Visual capture attempts by result -- a browserless outage is otherwise invisible while it silently degrades screenshots to dash cells, and the screenshot gate treats absent evidence as a close signal (#9487).", type: "counter" }], ["loopover_webhook_dedup_total", { help: "Webhook deliveries deduplicated before enqueue.", type: "counter" }], ["loopover_webhook_enqueue_total", { help: "Webhook enqueue outcomes by event and action.", type: "counter" }], @@ -425,3 +425,64 @@ export function resetMetrics(): void { redactedRepoLabels.clear(); for (const [name, meta] of DEFAULT_METRIC_META) metricMeta.set(name, meta); } + +/** + * #9487: the FIXED set of route groups `loopover_http_request_duration_seconds` may be labelled with. + * + * The histogram had no route label at all, so `sum by (le, route) (...)` returned a single unlabelled series + * and a fired latency-SLO alert could not be attributed to anything. The reason it had none is the reason + * this list is an ALLOWLIST rather than a path sanitizer: the label value must be bounded by CONSTRUCTION, + * because a caller-controlled path (or any per-PR/per-repo id inside one) is unbounded cardinality — the one + * failure that takes a Prometheus down rather than merely misinforming it. + * + * Groups are the first path segment under `/v1` plus the handful of top-level surfaces, chosen because that + * is the granularity an operator actually acts on ("the webhook lane is slow", "MCP is slow"). Anything not + * listed collapses to `other` — including every 404 — so an attacker probing random paths adds exactly zero + * new series. + */ +const HTTP_ROUTE_GROUPS: ReadonlySet = new Set([ + "agent", + "ams", + "app", + "auth", + "bounties", + "contributors", + "drafts", + "enrichment-analyzers", + "finding-taxonomy", + "github", + "installations", + "internal", + "issue-rag", + "lint", + "local", + "loop", + "mcp", + "opportunities", + "orb", + "preflight", + "public", + "repos", + "reviews", + "settings", + "signals", + "stats", + "webhooks", +]); + +/** + * A bounded, low-cardinality group for `path` — never the raw path. Returns one of {@link HTTP_ROUTE_GROUPS}, + * `loopover` for the public shot/asset surface, or `other`. PURE. + * + * Deliberately NOT a "replace ids with :param" normalizer: that still trusts the path's SHAPE, and one + * unmatched route pattern reintroduces unbounded cardinality silently. An allowlist can only ever be wrong in + * the safe direction (a real route lands in `other` until someone adds it). + */ +export function httpRouteGroup(path: string): string { + if (path.startsWith("/loopover/")) return "loopover"; + if (!path.startsWith("/v1/")) return "other"; + /* v8 ignore next -- String.split always yields at least one element (even "".split("/") is [""]), so the + nullish arm is unreachable; it exists only to satisfy noUncheckedIndexedAccess. */ + const segment = path.slice("/v1/".length).split("/")[0] ?? ""; + return HTTP_ROUTE_GROUPS.has(segment) ? segment : "other"; +} diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index 44fe3b9fb..e9ced3dc0 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -139,7 +139,7 @@ async function retryPoolUpdateOrLeaveForReclaim( return null; } } -import { hostLoadAvg1PerCore } from "./host-pressure"; +import { hostLoadAvg1PerCore, hostMemoryUsedFraction } from "./host-pressure"; import { evaluateMaintenanceAdmission, isMaintenanceAdmissionGrantedUnderPressure, @@ -531,6 +531,7 @@ export function createPgQueue( backlogConvergencePendingCount: Number(backlogConvergence.cnt), freshIntakePendingCount: Number(freshIntake.cnt), hostLoadAvg1PerCore: hostLoadAvg1PerCore(), + hostMemoryUsedFraction: hostMemoryUsedFraction(), }; } diff --git a/src/selfhost/process-lifecycle.ts b/src/selfhost/process-lifecycle.ts index 768718feb..c892321ad 100644 --- a/src/selfhost/process-lifecycle.ts +++ b/src/selfhost/process-lifecycle.ts @@ -47,12 +47,21 @@ export type InstallSelfHostCrashHandlersOptions = { * `process.exit()` tears the event loop down immediately otherwise, which would make capture a near-total * no-op in practice for a batching client. No-op default. Never expected to throw/reject. */ flush?: () => Promise; + /** #9487: hard ceiling on how long the flush above may delay the exit. Injectable so a test can drive the + * deadline path deterministically instead of waiting out the real one. */ + flushDeadlineMs?: number; /** Reinstall even if handlers were already installed (mainly for tests). */ force?: boolean; }; let handlersInstalled = false; +/** #9487: how long a fatal handler may wait on telemetry flush before exiting anyway. This handler exists to + * GUARANTEE the restart; a wedged egress must not be able to hold a process that has already declared itself + * unsound. Three seconds is generous for a batching HTTP client and still far inside any restart supervisor's + * patience. */ +const FATAL_FLUSH_DEADLINE_MS = 3_000; + /** Render any thrown/rejected value as a single log-safe string, preferring an Error's stack -- mirrors the * miner's own describeError exactly. */ function describeError(value: unknown): string { @@ -74,6 +83,7 @@ export function installSelfHostCrashHandlers(options: InstallSelfHostCrashHandle const exit = typeof options.exit === "function" ? options.exit : (code: number) => proc.exit(code); const captureError = typeof options.captureError === "function" ? options.captureError : () => {}; const flush = typeof options.flush === "function" ? options.flush : async () => {}; + const flushDeadlineMs = typeof options.flushDeadlineMs === "number" ? options.flushDeadlineMs : FATAL_FLUSH_DEADLINE_MS; if (handlersInstalled && options.force !== true) return false; handlersInstalled = true; @@ -87,7 +97,21 @@ export function installSelfHostCrashHandlers(options: InstallSelfHostCrashHandle }), ); captureError(error, { kind }); - await flush(); + // #9487: bounded. This handler exists to GUARANTEE the restart, and an unbounded `await flush()` let a + // wedged telemetry egress (PostHog, via server.ts's flushPostHog) delay or entirely prevent the very exit + // it is here to perform -- a process that has already logged a fatal sitting alive indefinitely, serving + // requests from a state it declared unsound. Losing a few telemetry events is unambiguously the cheaper + // failure. `race` (not a cancel) because there is nothing to cancel: the flush promise is abandoned and + // the process exits under it. + await Promise.race([ + flush(), + new Promise((resolve) => { + // `unref()` so a pending deadline timer can never itself hold the process open past a flush that + // resolved first -- the timer exists to bound the wait, not to extend the lifetime it is bounding. + const timer = setTimeout(resolve, flushDeadlineMs); + (timer as unknown as { unref?: () => void }).unref?.(); + }), + ]); exit(1); }; diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 6b1bf7eaf..66dba5e92 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -46,7 +46,7 @@ import { type GitHubRateLimitAdmissionTarget, type SelfHostQueueSnapshot, } from "./queue-common"; -import { hostLoadAvg1PerCore } from "./host-pressure"; +import { hostLoadAvg1PerCore, hostMemoryUsedFraction } from "./host-pressure"; import { evaluateMaintenanceAdmission, isMaintenanceAdmissionGrantedUnderPressure, @@ -1631,6 +1631,7 @@ function maintenancePressureSignals(driver: SqliteDriver, now: number): Maintena backlogConvergencePendingCount: Number(backlogConvergence.cnt), freshIntakePendingCount: Number(freshIntake.cnt), hostLoadAvg1PerCore: hostLoadAvg1PerCore(), + hostMemoryUsedFraction: hostMemoryUsedFraction(), }; } diff --git a/src/server.ts b/src/server.ts index 2ecb7563c..adf84a76d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -51,6 +51,7 @@ import { loadFileSecrets } from "./selfhost/load-file-secrets"; import { backupAcknowledgedGaugeValue, buildHealthBody, + browserEndpointReadinessProbe, codexAuthReadinessProbe, emptyConfigDirAcknowledgedGaugeValue, emptyConfigDirAdvisory, @@ -63,7 +64,7 @@ import { } from "./selfhost/health"; import { clockSkewSampleAgeSeconds, clockSkewSecondsSample } from "./selfhost/clock-skew"; import { d1DatabaseSizeBytesSample, d1SignalSnapshotsRowsPerKeySample, d1TableRowCountSamples, isD1SizeProbeEnabled, runD1SizeProbe } from "./selfhost/d1-size-probe"; -import { gauge, gaugeVector, incr, observe, renderMetrics, setSelfHostedMetricsMode, setSelfHostedRawRepoLabels } from "./selfhost/metrics"; +import { gauge, gaugeVector, httpRouteGroup, incr, observe, renderMetrics, setSelfHostedMetricsMode, setSelfHostedRawRepoLabels } from "./selfhost/metrics"; import { CRON_INTERVAL_MIN_MS, delayToNextWallClockBoundaryMs } from "./selfhost/cron-alignment"; import { runSelfHostMigrations } from "./selfhost/migrate"; import { createPgAdapter, tuneGithubRateLimitObservationsAutovacuum, widenGithubIdColumnsToBigint } from "./selfhost/pg-adapter"; @@ -912,6 +913,20 @@ async function main(): Promise { }); } + // #9487/#9464: browserless readiness. A visual-capture outage used to be invisible in /ready and in + // Prometheus while it silently affected gate outcomes; this makes it observable at the same place every + // other optional backend is. No-op unless BROWSER_WS_ENDPOINT is configured. + const browserProbe = browserEndpointReadinessProbe(process.env, async (url) => { + const response = await fetch(url, { signal: AbortSignal.timeout(1500) }); + return { ok: response.ok }; + }); + if (browserProbe) { + readinessProbes.push({ + name: browserProbe.name, + check: () => withTimeout(browserProbe.check()), + }); + } + gauge("loopover_queue_pending", () => backend.queue.size()); gauge("loopover_queue_dead", () => backend.queue.deadCount()); // #9139: pass backend.queue's own recentDeadCount so this reads the self-host dead-letter path (the @@ -1197,9 +1212,12 @@ async function main(): Promise { incr("loopover_http_requests_total", { status: `${Math.floor(response.status / 100)}xx`, }); + // #9487: labelled by BOUNDED route group, never the raw path -- a fired latency-SLO alert + // was previously unattributable, because this histogram had a single unlabelled series. observe( "loopover_http_request_duration_seconds", (Date.now() - startedReq) / 1000, + { route: httpRouteGroup(path) }, ); setCurrentOtelSpanAttributes(selfHostHttpResponseAttributes(response.status)); return response; diff --git a/test/unit/selfhost-blob-store.test.ts b/test/unit/selfhost-blob-store.test.ts index db91b4231..d49218c38 100644 --- a/test/unit/selfhost-blob-store.test.ts +++ b/test/unit/selfhost-blob-store.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -78,3 +78,72 @@ describe("createFsBlobStore (#10 — self-host visual screenshot persistence)", await expect(createFsBlobStore(dir).delete("loopover/shots/never-existed.png")).resolves.toBeUndefined(); }); }); + +// #9487: keys here are INPUT-HASH-ADDRESSED (`loopover/shots/.png`), so a half-written file from a +// mid-write kill is never retried or overwritten — the next lookup for that same input finds a file and +// serves a truncated PNG as a permanently "valid" cache entry. tmp+rename makes a reader see either no file +// or the complete one; the config writer (private-config.ts's atomicWriteWithBackup) already did this. +// +// HONEST SCOPE: the primary benefit is crash-safety (SIGKILL between the first and last byte), which cannot +// be simulated in-process — nothing here can kill the writer mid-`writeFile`. These tests pin what IS +// observable: that the mechanism is tmp+rename rather than a direct write, that no temp files survive either +// outcome, and that concurrent writers of one key cannot publish each other's bytes. +describe("atomic writes (#9487)", () => { + /** A body that yields some bytes and then errors — the shape of a source dying mid-write. */ + const erroringStream = (): ReadableStream => + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([0x89, 0x50])); + controller.error(new Error("source died mid-write")); + }, + }); + + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "gitt-blob-atomic-")); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + // The mechanism itself, pinned at the source. A direct `writeFile(target, ...)` is the bug; without this + // assertion nothing in this file would notice a revert, because the crash window it protects is exactly the + // part an in-process test cannot reach. + it("REGRESSION: writes via a unique temp path and renames — never a direct write to the target key", () => { + const source = readFileSync("src/selfhost/blob-store.ts", "utf8"); + expect(source).toContain("await rename(tmpTarget, target)"); + expect(source).toContain(".tmp-${randomUUID()}"); + expect(source).not.toContain("await writeFile(target,"); + }); + + it("INVARIANT: a successful put leaves the object and NO temp leftovers", async () => { + const store = createFsBlobStore(dir); + await store.put("loopover/shots/abc.png", new Uint8Array([0x89, 0x50, 0x4e, 0x47])); + + expect(readdirSync(join(dir, "loopover", "shots"))).toEqual(["abc.png"]); + }); + + it("INVARIANT: a failed put leaves no object at the key and no temp leftovers", async () => { + const store = createFsBlobStore(dir); + await expect(store.put("loopover/shots/broken.png", erroringStream())).rejects.toThrow(); + + expect(await store.get("loopover/shots/broken.png")).toBeNull(); + const shotsDir = join(dir, "loopover", "shots"); + expect(existsSync(shotsDir) ? readdirSync(shotsDir) : []).toEqual([]); + }); + + it("INVARIANT: two concurrent puts of the SAME key both complete, and the object is one of them INTACT", async () => { + // Unique temp names are what make this safe: a shared temp path would let one writer's rename publish the + // other's partially-written file. + const store = createFsBlobStore(dir); + const a = new Uint8Array([1, 1, 1, 1, 1, 1, 1, 1]); + const b = new Uint8Array([2, 2, 2, 2]); + await Promise.all([store.put("loopover/shots/race.png", a), store.put("loopover/shots/race.png", b)]); + + const obj = await store.get("loopover/shots/race.png"); + const bytes = Array.from(new Uint8Array(await new Response(obj!.body).arrayBuffer())); + expect([Array.from(a), Array.from(b)]).toContainEqual(bytes); // intact, never interleaved or truncated + expect(readdirSync(join(dir, "loopover", "shots"))).toEqual(["race.png"]); + }); +}); diff --git a/test/unit/selfhost-health.test.ts b/test/unit/selfhost-health.test.ts index 0a49340b9..78eebaf75 100644 --- a/test/unit/selfhost-health.test.ts +++ b/test/unit/selfhost-health.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it, vi } from "vitest"; import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; import { backupAcknowledgedGaugeValue, + browserEndpointReadinessProbe, buildHealthBody, codexAuthReadinessProbe, emptyConfigDirAcknowledgedGaugeValue, @@ -458,3 +459,87 @@ describe("readiness (#982)", () => { }); }); }); + +// #9487/#9464: every other dependency the review path needs has a readiness probe (Redis, Qdrant, the GitHub +// App, codex); BROWSER_WS_ENDPOINT had none. A browserless outage was therefore invisible in /ready AND in +// Prometheus while it silently changed gate OUTCOMES — the screenshot-table gate reads absent visual evidence +// as a close signal. #9464 stopped that from closing PRs; this makes the outage itself visible rather than +// inferred afterwards. +describe("browserEndpointReadinessProbe (#9487)", () => { + it("is not registered at all when the endpoint is unconfigured — a deployment without visual review is not degraded", () => { + expect(browserEndpointReadinessProbe({}, async () => ({ ok: true }))).toBeNull(); + expect(browserEndpointReadinessProbe({ BROWSER_WS_ENDPOINT: " " }, async () => ({ ok: true }))).toBeNull(); + }); + + it("REGRESSION: reports ready when the endpoint answers, and NOT ready when it does not", async () => { + const up = browserEndpointReadinessProbe({ BROWSER_WS_ENDPOINT: "ws://browserless:3000" }, async () => ({ ok: true })); + await expect(up!.check()).resolves.toBe(true); + + const down = browserEndpointReadinessProbe({ BROWSER_WS_ENDPOINT: "ws://browserless:3000" }, async () => ({ ok: false })); + await expect(down!.check()).resolves.toBe(false); + + const unreachable = browserEndpointReadinessProbe({ BROWSER_WS_ENDPOINT: "ws://browserless:3000" }, async () => { + throw new Error("ECONNREFUSED"); + }); + await expect(unreachable!.check()).resolves.toBe(false); + }); + + it("probes the HTTP sibling of the ws endpoint, dropping any query string", async () => { + // Cheap enough for a poll loop (no websocket, no Chromium launch). The token must never reach a probe URL + // that could be logged — and /json/version needs no auth. + const seen: string[] = []; + const probe = browserEndpointReadinessProbe({ BROWSER_WS_ENDPOINT: "ws://browserless:3000?token=secret-token" }, async (url) => { + seen.push(url); + return { ok: true }; + }); + await probe!.check(); + expect(seen).toEqual(["http://browserless:3000/json/version"]); + expect(seen[0]).not.toContain("secret-token"); + }); + + it("maps wss:// to https://", async () => { + const seen: string[] = []; + const probe = browserEndpointReadinessProbe({ BROWSER_WS_ENDPOINT: "wss://browser.example.com" }, async (url) => { + seen.push(url); + return { ok: true }; + }); + await probe!.check(); + expect(seen).toEqual(["https://browser.example.com/json/version"]); + }); + + it("INVARIANT: a configured-but-unparseable endpoint fails readiness CLOSED rather than silently skipping", async () => { + for (const bad of ["http://browserless:3000", "not-a-url", "://nope"]) { + const probe = browserEndpointReadinessProbe({ BROWSER_WS_ENDPOINT: bad }, async () => ({ ok: true })); + expect(probe, bad).not.toBeNull(); + await expect(probe!.check(), bad).resolves.toBe(false); + } + }); + + it("INVARIANT: caches within the TTL and single-flights concurrent checks, so /ready polling cannot itself load a struggling backend", async () => { + let calls = 0; + const probe = browserEndpointReadinessProbe({ BROWSER_WS_ENDPOINT: "ws://browserless:3000" }, async () => { + calls += 1; + return { ok: true }; + }); + await Promise.all([probe!.check(), probe!.check(), probe!.check()]); + await probe!.check(); + expect(calls).toBe(1); + }); + + it("INVARIANT: re-probes once the TTL lapses, so a recovered backend is not pinned unhealthy", async () => { + let ok = false; + let calls = 0; + const probe = browserEndpointReadinessProbe( + { BROWSER_WS_ENDPOINT: "ws://browserless:3000" }, + async () => { + calls += 1; + return { ok }; + }, + 0, // TTL 0 ⇒ every check re-probes + ); + await expect(probe!.check()).resolves.toBe(false); + ok = true; + await expect(probe!.check()).resolves.toBe(true); + expect(calls).toBe(2); + }); +}); diff --git a/test/unit/selfhost-host-pressure.test.ts b/test/unit/selfhost-host-pressure.test.ts index 557ad8b5a..07a86ed2a 100644 --- a/test/unit/selfhost-host-pressure.test.ts +++ b/test/unit/selfhost-host-pressure.test.ts @@ -3,6 +3,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; vi.mock("node:os", () => ({ loadavg: vi.fn(), cpus: vi.fn(), + totalmem: vi.fn(), + freemem: vi.fn(), })); describe("hostLoadAvg1PerCore", () => { @@ -78,3 +80,48 @@ describe("hostLoadAvg1PerCore", () => { expect(hostLoadAvg1PerCore()).toBeNull(); }); }); + +// #9487: host pressure watched CPU only. On the box this was found on — Ollama resident at ~9.9 GiB +// alongside browserless at ~1.5 GiB — memory is the realistic killer, and nothing observed it: the OOM +// killer made the call instead, taking the whole container and every in-flight job with it rather than +// deferring one maintenance job. +describe("hostMemoryUsedFraction (#9487)", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.resetModules(); + }); + + const withMemory = async (total: number, free: number) => { + const os = await import("node:os"); + vi.mocked(os.totalmem).mockReturnValue(total); + vi.mocked(os.freemem).mockReturnValue(free); + const { hostMemoryUsedFraction } = await import("../../src/selfhost/host-pressure"); + return hostMemoryUsedFraction(); + }; + + it("reports the used fraction of host memory", async () => { + expect(await withMemory(16_000_000_000, 4_000_000_000)).toBeCloseTo(0.75, 5); + expect(await withMemory(16_000_000_000, 16_000_000_000)).toBe(0); + expect(await withMemory(16_000_000_000, 0)).toBe(1); + }); + + it("INVARIANT: an impossible reading yields null (signal unavailable), never a misleading 0", async () => { + // Fail-open, same contract as hostLoadAvg1PerCore: a caller treats null as "skip this check". Clamping + // instead would let a broken platform reading masquerade as "no pressure", which is the one answer that + // must never be fabricated. + expect(await withMemory(0, 0)).toBeNull(); // no total + expect(await withMemory(Number.NaN, 1)).toBeNull(); + expect(await withMemory(16_000_000_000, Number.NaN)).toBeNull(); + expect(await withMemory(16_000_000_000, -1)).toBeNull(); + expect(await withMemory(16_000_000_000, 32_000_000_000)).toBeNull(); // free > total ⇒ ratio out of range + }); + + it("INVARIANT: a throwing os reading degrades to null rather than propagating", async () => { + const os = await import("node:os"); + vi.mocked(os.totalmem).mockImplementation(() => { + throw new Error("platform boom"); + }); + const { hostMemoryUsedFraction } = await import("../../src/selfhost/host-pressure"); + expect(hostMemoryUsedFraction()).toBeNull(); + }); +}); diff --git a/test/unit/selfhost-load-file-secrets.test.ts b/test/unit/selfhost-load-file-secrets.test.ts index 91104b2de..01a5da910 100644 --- a/test/unit/selfhost-load-file-secrets.test.ts +++ b/test/unit/selfhost-load-file-secrets.test.ts @@ -104,3 +104,53 @@ describe("loadFileSecrets (#4403)", () => { } }); }); + +// #9487: a missing/unreadable secret file correctly throws (#6284), but a zero-byte or whitespace-only one +// set `env[NAME] = ""` — which every downstream `nonBlank()` reads as UNCONFIGURED, and preflight.ts +// deliberately skips absent values. So a truncated GITHUB_WEBHOOK_SECRET file booted an instance that +// rejected every webhook, healthily, forever. Directly adjacent to the known rotation footgun on edge-nl-01, +// where the file is rewritten in place: the window in which it is momentarily empty is exactly when a +// container restart reads it. +describe("empty secret files are fatal, not silently unconfigured (#9487)", () => { + it.each([ + ["zero-byte", ""], + ["whitespace-only", " \n\t "], + ])("REGRESSION: a %s secret file throws instead of setting an empty value", (_label, contents) => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const env: Record = { GITHUB_WEBHOOK_SECRET_FILE: "/run/secrets/webhook" }; + + expect(() => loadFileSecrets(env, () => contents)).toThrow(/empty/i); + // The decisive assertion: the target must be left UNSET, never "" — an empty string is precisely the + // value that reads as "not configured" downstream while looking like a successful load here. + expect(env.GITHUB_WEBHOOK_SECRET).toBeUndefined(); + expect(errorSpy.mock.calls.some((call) => String(call[0]).includes("selfhost_secret_file_empty"))).toBe(true); + errorSpy.mockRestore(); + }); + + it("INVARIANT: the empty and unreadable failures stay DISTINCT — an operator must know which problem they have", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const unreadable: Record = { A_SECRET_FILE: "/nope" }; + expect(() => + loadFileSecrets(unreadable, () => { + throw new Error("ENOENT"); + }), + ).toThrow(/Failed to read secret file/); + expect(errorSpy.mock.calls.some((call) => String(call[0]).includes("selfhost_secret_file_unreadable"))).toBe(true); + expect(errorSpy.mock.calls.some((call) => String(call[0]).includes("selfhost_secret_file_empty"))).toBe(false); + errorSpy.mockRestore(); + }); + + it("INVARIANT: a secret file with surrounding whitespace still loads, trimmed — only a FULLY empty one is fatal", () => { + const env: Record = { A_SECRET_FILE: "/run/secrets/a" }; + loadFileSecrets(env, () => " real-value\n"); + expect(env.A_SECRET).toBe("real-value"); + }); + + it("INVARIANT: an explicit env value still wins and the file is never read, empty or not", () => { + const readFile = vi.fn(() => ""); + const env: Record = { A_SECRET: "explicit", A_SECRET_FILE: "/run/secrets/a" }; + loadFileSecrets(env, readFile); + expect(env.A_SECRET).toBe("explicit"); + expect(readFile).not.toHaveBeenCalled(); + }); +}); diff --git a/test/unit/selfhost-maintenance-admission.test.ts b/test/unit/selfhost-maintenance-admission.test.ts index c67cd4b5e..12786db8d 100644 --- a/test/unit/selfhost-maintenance-admission.test.ts +++ b/test/unit/selfhost-maintenance-admission.test.ts @@ -21,6 +21,7 @@ const CLEAR_SIGNALS: MaintenancePressureSignals = { backlogConvergencePendingCount: 0, freshIntakePendingCount: 0, hostLoadAvg1PerCore: null, + hostMemoryUsedFraction: null, }; const CONFIG: MaintenanceAdmissionConfig = { @@ -29,6 +30,7 @@ const CONFIG: MaintenanceAdmissionConfig = { maxLiveJobAgeMs: 120_000, maxMaintenancePendingCount: 15, maxHostLoadAvg1PerCore: 1.5, + maxHostMemoryUsedFraction: 0.92, maxBacklogConvergencePendingCount: 10, deferMs: 180_000, maxDeferAgeMs: 4 * 60 * 60_000, @@ -412,6 +414,7 @@ describe("resolveMaintenanceAdmissionConfig", () => { maxLiveJobAgeMs: 120_000, maxMaintenancePendingCount: 15, maxHostLoadAvg1PerCore: 1.5, + maxHostMemoryUsedFraction: 0.92, maxBacklogConvergencePendingCount: 10, deferMs: 180_000, maxDeferAgeMs: 4 * 60 * 60_000, @@ -490,3 +493,66 @@ describe("resolveMaintenanceAdmissionConfig", () => { expect(resolveMaintenanceAdmissionConfig().maxHostLoadAvg1PerCore).toBe(1.5); }); }); + +// #9487: memory joins CPU as a peer pressure dimension. Deferring one maintenance job is unambiguously +// cheaper than an OOM kill, which takes the container and every in-flight job with it. +describe("host memory pressure (#9487)", () => { + it("REGRESSION: defers maintenance when host memory is above the threshold, with its own distinct reason", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, hostMemoryUsedFraction: 0.97 }, + CONFIG, + 0, + 0, + ); + expect(decision).toEqual({ admit: false, reason: "host_memory_high" }); + }); + + it("INVARIANT: memory at or below the threshold admits — the gate is strictly-greater, not >=", () => { + for (const fraction of [0.92, 0.5, 0]) { + expect( + evaluateMaintenanceAdmission({ ...CLEAR_SIGNALS, hostMemoryUsedFraction: fraction }, CONFIG, 0, 0), + ).toEqual({ admit: true, reason: "pressure_clear" }); + } + }); + + it("INVARIANT: a null reading SKIPS the check — an unavailable signal must never itself defer work", () => { + expect( + evaluateMaintenanceAdmission({ ...CLEAR_SIGNALS, hostMemoryUsedFraction: null }, CONFIG, 0, 0), + ).toEqual({ admit: true, reason: "pressure_clear" }); + }); + + it("REGRESSION: memory also gates the maintenance-drain ESCAPE — draining onto a near-OOM box is what the signal exists to prevent", () => { + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, maintenancePendingCount: CONFIG.maxMaintenancePendingCount + 1, hostMemoryUsedFraction: 0.97 }, + CONFIG, + Date.now() - CONFIG.maintenanceDrainAgeMs, + Date.now(), + ); + expect(decision).toEqual({ admit: false, reason: "host_memory_high" }); + }); + + it("INVARIANT: CPU still outranks memory, so an existing host_load_high denial keeps its reason", () => { + // Both dimensions over threshold: the reason must stay stable rather than flip with evaluation order, + // since operators alert and dashboard on these reason strings. + const decision = evaluateMaintenanceAdmission( + { ...CLEAR_SIGNALS, hostLoadAvg1PerCore: 99, hostMemoryUsedFraction: 0.99 }, + CONFIG, + 0, + 0, + ); + expect(decision).toEqual({ admit: false, reason: "host_load_high" }); + }); + + it("INVARIANT: MAINTENANCE_ADMISSION_MAX_HOST_MEMORY overrides the default, and a junk value falls back", () => { + const original = process.env.MAINTENANCE_ADMISSION_MAX_HOST_MEMORY; + try { + process.env.MAINTENANCE_ADMISSION_MAX_HOST_MEMORY = "0.5"; + expect(resolveMaintenanceAdmissionConfig().maxHostMemoryUsedFraction).toBe(0.5); + process.env.MAINTENANCE_ADMISSION_MAX_HOST_MEMORY = "not-a-number"; + expect(resolveMaintenanceAdmissionConfig().maxHostMemoryUsedFraction).toBe(0.92); + } finally { + if (original === undefined) delete process.env.MAINTENANCE_ADMISSION_MAX_HOST_MEMORY; + else process.env.MAINTENANCE_ADMISSION_MAX_HOST_MEMORY = original; + } + }); +}); diff --git a/test/unit/selfhost-metrics.test.ts b/test/unit/selfhost-metrics.test.ts index a4c3f761b..5fc58635b 100644 --- a/test/unit/selfhost-metrics.test.ts +++ b/test/unit/selfhost-metrics.test.ts @@ -8,6 +8,7 @@ import { gauge, gaugeVector, hitRatio, + httpRouteGroup, incr, observe, registerMetricMeta, @@ -498,3 +499,54 @@ describe("DEFAULT_METRIC_META completeness (drift guard, 2026-07 fix)", () => { expect(missing).toEqual([]); }); }); + +// #9487: LoopoverRequestLatencySLOBreach fired at p95 = 9.75s against a 1s SLO and was UNACTIONABLE — the +// histogram carried no route label, so `sum by (le, route) (...)` returned one unlabelled series and the +// alert could not be attributed to anything. The reason it had no label is exactly why this one is an +// allowlist: an unbounded label value is the failure that takes Prometheus down, not merely misinforms it. +describe("httpRouteGroup (#9487)", () => { + it("groups a /v1 route by its first segment", () => { + expect(httpRouteGroup("/v1/github/webhook")).toBe("github"); + expect(httpRouteGroup("/v1/mcp")).toBe("mcp"); + expect(httpRouteGroup("/v1/public/stats")).toBe("public"); + expect(httpRouteGroup("/v1/repos/owner/name/agent/pending-actions")).toBe("repos"); + }); + + it("groups the public asset/shot surface", () => { + expect(httpRouteGroup("/loopover/shot?key=abc")).toBe("loopover"); + }); + + it("REGRESSION: an UNKNOWN path is always `other` — cardinality is bounded by construction, not by sanitizing", () => { + // The decisive property. A caller-controlled path (or an id embedded in one) must add zero new series, + // so probing random paths cannot grow the metric at all. + for (const path of [ + "/", + "/nope", + "/v1/", + "/v1/not-a-real-surface/x", + "/v1/github-but-not-really", + "/v2/github/webhook", + "/v1/../../etc/passwd", + `/v1/${"x".repeat(5000)}`, + "/loopoverx/shot", + ]) { + expect(httpRouteGroup(path), path).toBe("other"); + } + }); + + it("INVARIANT: the group set is finite, so the label can only ever take a known value", () => { + // Fuzz over random paths: every result must be a value the allowlist could produce. This is the property + // that makes the label safe to ship on a per-request histogram at all. + const seen = new Set(); + for (let i = 0; i < 500; i += 1) { + seen.add(httpRouteGroup(`/v1/${Math.random().toString(36).slice(2)}/${i}`)); + seen.add(httpRouteGroup(`/${Math.random().toString(36).slice(2)}`)); + } + expect([...seen]).toEqual(["other"]); + }); + + it("INVARIANT: a per-PR path never leaks an id into the label", () => { + expect(httpRouteGroup("/v1/repos/acme/widgets/pulls/12345/review")).toBe("repos"); + expect(httpRouteGroup("/v1/orb/instances/inst-abc-123/status")).toBe("orb"); + }); +}); diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index e833ff5c5..8ca1b757d 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -16,7 +16,9 @@ import type { JobMessage } from "../../src/types"; // Real host CPU load is nondeterministic (and can legitimately spike on a busy CI runner), so every // maintenance-admission test in this file would be flaky against the real node:os signal. Default to // "unavailable" (null, never gates) here; individual host-load tests override the mock explicitly. -vi.mock("../../src/selfhost/host-pressure", () => ({ hostLoadAvg1PerCore: vi.fn(() => null) })); +// #9487: hostMemoryUsedFraction joined this module as a peer pressure signal — a partial mock would leave it +// undefined at runtime, which reads as "unavailable" only by accident. +vi.mock("../../src/selfhost/host-pressure", () => ({ hostLoadAvg1PerCore: vi.fn(() => null), hostMemoryUsedFraction: vi.fn(() => null) })); // The PG connection-resilience tests below deliberately trigger retryPoolQuery's real ECONNRESET retry // path; collapse its per-attempt backoff to near-zero so they don't pay real wall-clock time for it @@ -4136,6 +4138,7 @@ describe("createPgQueue (durable #977)", () => { backlogConvergencePendingCount: 3, freshIntakePendingCount: 5, hostLoadAvg1PerCore: null, + hostMemoryUsedFraction: null, }); }); diff --git a/test/unit/selfhost-process-lifecycle.test.ts b/test/unit/selfhost-process-lifecycle.test.ts index 39c263c6d..6514eb586 100644 --- a/test/unit/selfhost-process-lifecycle.test.ts +++ b/test/unit/selfhost-process-lifecycle.test.ts @@ -176,3 +176,66 @@ describe("self-host process crash handlers (#9133)", () => { expect(logged.error.length).toBe(4000); }); }); + +// #9487: this handler exists to GUARANTEE the restart, and it awaited `flush()` with no deadline before +// exit(1). A wedged telemetry egress (PostHog, via server.ts's flushPostHog) therefore delayed — or entirely +// prevented — the very exit the module is here to perform, leaving a process that has already logged a fatal +// alive and serving from a state it declared unsound. Losing a few telemetry events is the cheaper failure. +describe("fatal flush is bounded (#9487)", () => { + beforeEach(() => { + resetSelfHostCrashHandlersForTest(); + }); + + it("REGRESSION: exits even when flush never settles", async () => { + const { proc, handlers, exit } = makeFakeProcess(); + installSelfHostCrashHandlers({ + process: proc, + log: () => {}, + exit, + flush: () => new Promise(() => {}), // never settles, the wedged-egress case + flushDeadlineMs: 1, + force: true, + }); + + await handlers.get("uncaughtException")!(new Error("boom")); + + expect(exit).toHaveBeenCalledWith(1); + }); + + it("INVARIANT: a flush that settles first is still awaited — the deadline is a ceiling, not a truncation", async () => { + const { proc, handlers, exit } = makeFakeProcess(); + let flushed = false; + installSelfHostCrashHandlers({ + process: proc, + log: () => {}, + exit, + flush: async () => { + flushed = true; + }, + flushDeadlineMs: 60_000, // far longer than the flush takes + force: true, + }); + + await handlers.get("unhandledRejection")!(new Error("boom")); + + expect(flushed).toBe(true); + expect(exit).toHaveBeenCalledWith(1); + }); + + it("INVARIANT: both fatal events are bounded identically — neither can hang the restart", async () => { + for (const event of ["uncaughtException", "unhandledRejection"] as const) { + resetSelfHostCrashHandlersForTest(); + const { proc, handlers, exit } = makeFakeProcess(); + installSelfHostCrashHandlers({ + process: proc, + log: () => {}, + exit, + flush: () => new Promise(() => {}), + flushDeadlineMs: 1, + force: true, + }); + await handlers.get(event)!(new Error("boom")); + expect(exit, event).toHaveBeenCalledWith(1); + } + }); +}); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 54bb473d8..72e778b3f 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -16,7 +16,9 @@ import type { JobMessage } from "../../src/types"; // Real host CPU load is nondeterministic (and can legitimately spike on a busy CI runner), so every // maintenance-admission test in this file would be flaky against the real node:os signal. Default to // "unavailable" (null, never gates) here; individual host-load tests override the mock explicitly. -vi.mock("../../src/selfhost/host-pressure", () => ({ hostLoadAvg1PerCore: vi.fn(() => null) })); +// #9487: hostMemoryUsedFraction joined this module as a peer pressure signal — a partial mock would leave it +// undefined at runtime, which reads as "unavailable" only by accident. +vi.mock("../../src/selfhost/host-pressure", () => ({ hostLoadAvg1PerCore: vi.fn(() => null), hostMemoryUsedFraction: vi.fn(() => null) })); function makeDriver(): ReturnType { return nodeSqliteDriver(new DatabaseSync(":memory:") as never);