fix(ledger,retention): stop retention manufacturing tamper signals, and make the public counter durable (#9474, #9489)

apps/loopover-ui/content/docs/what-you-can-verify.mdx

@@ -29,8 +29,24 @@ hash, so any edit to history breaks the chain at a point you can locate.
 curl -s "https://api.loopover.ai/v1/public/decision-ledger/verify" | jq
 ```
 
-Returns `{ ok, checked, nextAfterSeq, tipSeq, tipHash, totalCount }`, and a `break` object with a
-`409` status if the chain is inconsistent. No API key — **anyone** can run it.
+Returns `{ ok, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords }`, and a `break`
+object with a `409` status if the chain is inconsistent. No API key — **anyone** can run it.
+
+Two response fields encode deliberate, published semantics rather than tolerance for tampering:
+
+- **`prunedRecords`** — decision *records* (the preimages) are retained for 180 days; ledger rows are
+  kept forever. A ledger row whose hash-chained `createdAt` is older than that window may therefore
+  reference a pruned record: the chain checks still hold for it, only the content re-check is
+  impossible, and it is counted here instead of reported as `missing_record`. The tolerance keys on
+  the *ledger row's* timestamp, which is inside the hash chain — backdating it to sneak a fresh
+  deletion under the window breaks `row_hash_mismatch` first. What pruning genuinely gives up: the
+  row's committed digest stays published, but only a challenger holding the original preimage can
+  still prove a historical rewrite of it.
+- **Append grace (5 minutes)** — a record and its chain row are two writes moments apart, so a
+  record younger than the grace window with no chain entry is "append in flight", not a break. Older
+  than that, it is reported: past the verified tip as `short_tail` (the truncated-tail signature),
+  or behind it as `unchained_record` (the failed-append signature — interior orphans are found by an
+  anti-join over the whole record set, not just the tail).
 
 <Callout variant="note">
   **Trust assumption: tamper-evident, externally anchored.** The check above still catches sequence

apps/loopover-ui/public/openapi.json

@@ -20104,10 +20104,10 @@
         "summary": "Verify a window of the hash-chained decision ledger (resumable via afterSeq)",
         "responses": {
           "200": {
-            "description": "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing."
+            "description": "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)."
           },
           "409": {
-            "description": "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | short_tail (a record exists past the verified tip with no chain entry)"
+            "description": "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | missing_record | content_mismatch | short_tail (a record newer than the verified tip has no chain entry — the truncated-tail signature) | unchained_record (an INTERIOR record has no chain entry — the failed-append signature). Records younger than the 5-minute append grace window are not reported: the record insert and its chain append are two writes moments apart, and a verify landing between them is not evidence of tampering."
           }
         },
         "security": [

migrations/0198_orb_outcome_rollups.sql

@@ -0,0 +1,25 @@
+-- #9474: durable running totals for orb_pr_outcomes' CUMULATIVE public consumer.
+--
+-- #9415 gave orb_pr_outcomes a 90-day retention window, but getOrbGlobalStats SUMs the ENTIRE table and
+-- public-stats folds that into the homepage's all-time merged/closed/handled counters. Once rows began aging
+-- past 90 days (~2026-10-25 given #9415's merge date) the "all-time" numbers would have plateaued and then
+-- visibly DECREASED -- a published cumulative counter going backwards.
+--
+-- The retention prune now folds each about-to-be-deleted row into this table first, atomically (same batch
+-- transaction as the delete -- see pruneExpiredRecords' orb_pr_outcomes special case), and getOrbGlobalStats
+-- adds these totals to its live scan. Keyed per LOWERCASED account_login (matching the stats query's own
+-- LOWER() comparison; '' for a NULL login) so its excludeAccount de-dup keeps working after the raw rows are
+-- gone. Only rows the live query would have counted are folded: registered installations, no published
+-- review surface.
+CREATE TABLE IF NOT EXISTS orb_outcome_rollups (
+  account_login TEXT PRIMARY KEY,
+  merged INTEGER NOT NULL DEFAULT 0,
+  closed INTEGER NOT NULL DEFAULT 0,
+  total INTEGER NOT NULL DEFAULT 0,
+  updated_at TEXT NOT NULL
+);
+
+-- #9489/#9474: verifyDecisionLedger's completeness reconciliation now asks "does ANY ledger row vouch for
+-- this record" (a NOT EXISTS anti-join finding interior orphans, not just tail ones); without this index
+-- that is a full ledger scan per candidate record.
+CREATE INDEX IF NOT EXISTS decision_ledger_record_id ON decision_ledger (record_id);

scripts/check-schema-drift.ts

@@ -54,6 +54,9 @@ export const RAW_SQL_ONLY_TABLES: Set<string> = new Set([
   "orb_export_cursor",
   "orb_github_installations",
   "orb_instances",
+  // #9474: durable running totals folded from orb_pr_outcomes (itself raw-SQL-only, below) by the
+  // retention prune, and summed back in by getOrbGlobalStats -- both via env.DB.prepare, no Drizzle use.
+  "orb_outcome_rollups",
   "orb_pr_outcomes",
   "orb_relay_failures",
   "orb_reuse_counters",

src/db/retention.ts

@@ -14,6 +14,14 @@ export type RetentionRule = { table: string; column: string; days: number };
 const DURABLE_AUDIT_EVENT_TYPES = ["github_app.pr_public_surface_published"] as const;
 
 export const RETENTION_POLICY: readonly RetentionRule[] = [
+  // #9474: MUST stay ahead of audit_events. This table's prune first FOLDS the rows it is about to delete
+  // into the durable orb_outcome_rollups running totals (see pruneExpiredRecords' special case), and that
+  // fold reuses getOrbGlobalStats' exact counting semantics -- including the LEFT JOIN that excludes
+  // outcomes whose PR already published a review surface (a durable audit_events row). Both tables share a
+  // 90-day window, so if audit_events pruned first within the same pass, the very audit rows that exclusion
+  // needs would be gone by the time the fold ran, and the rollup would permanently over-count exactly the
+  // PRs the live query never counted.
+  { table: "orb_pr_outcomes", column: "occurred_at", days: 90 },
   { table: "audit_events", column: "created_at", days: 90 },
   { table: "ai_usage_events", column: "created_at", days: 90 },
   { table: "product_usage_events", column: "occurred_at", days: 180 },
@@ -74,7 +82,9 @@ export const RETENTION_POLICY: readonly RetentionRule[] = [
   { table: "pull_request_files", column: "updated_at", days: 30 },
   { table: "repo_github_totals_snapshots", column: "fetched_at", days: 30 },
   { table: "recent_merged_pull_requests", column: "updated_at", days: 30 },
-  { table: "orb_pr_outcomes", column: "occurred_at", days: 90 },
+  // orb_pr_outcomes was #9415's fifth entry here; #9474 moved it to the TOP of this policy (ordering
+  // constraint documented there) and gave it a fold-before-delete so the cumulative public counter it
+  // feeds can never shrink.
   // #9473: four more members of the same re-derivable/per-event class #9415 bounded, found by an audit sweep
   // for tables written per event with NO delete path anywhere in src/. Two have a pruned sibling, which is
   // what makes the omission clearly unintentional rather than a retention decision:
@@ -157,6 +167,20 @@ export const RETENTION_COMPOSITE_PK_TABLES: ReadonlySet<string> = new Set([
   "linked_issue_satisfaction_cache",
 ]);
 
+/**
+ * The retention cutoff for `table` as of `nowMs` -- rows with a timestamp strictly BELOW this are eligible
+ * for pruning -- or null when the table has no retention rule at all. #9474: exported so consumers whose
+ * correctness depends on a table's permanence can reason about its IMPERMANENCE instead of silently assuming.
+ * verifyDecisionLedger uses this to tell "this record was legitimately pruned by the published retention
+ * policy" apart from "this record is missing and should not be": the distinction is keyed on the LEDGER row's
+ * hash-chained created_at (which cannot be backdated without breaking the chain), so an operator cannot use
+ * the tolerance to hide a fresh deletion.
+ */
+export function retentionCutoffIsoForTable(table: string, nowMs: number = Date.parse(nowIso())): string | null {
+  const rule = RETENTION_POLICY.find((candidate) => candidate.table === table);
+  return rule ? cutoffIso(rule.days, nowMs) : null;
+}
+
 function pkColumnFor(table: string): string {
   return RETENTION_PK_COLUMN[table] ?? "rowid";
 }
@@ -219,6 +243,84 @@ export async function pruneExpiredRecords(
       continue;
     }
 
+    // #9474: orb_pr_outcomes feeds a CUMULATIVE public counter (getOrbGlobalStats -> the homepage "all-time"
+    // merged/closed totals), so its rows must be folded into the durable orb_outcome_rollups totals in the
+    // same transaction that deletes them -- a fold and delete that could commit separately would either
+    // double-count (fold landed, delete didn't, next run re-folds) or under-count (delete landed, fold
+    // didn't). One atomic batch, both statements scoped to the identical cutoff, sidesteps both. The delete
+    // is deliberately UNBATCHED for this one table: the aging cohort is one row per fleet-wide PR terminal
+    // per day (hundreds at most, vs the six-figure log tables the batching exists for), and a bounded delete
+    // would reintroduce the split-commit problem for whatever the bound left behind.
+    if (rule.table === "orb_pr_outcomes") {
+      // #9474: this table feeds a CUMULATIVE public counter (getOrbGlobalStats -> the homepage "all-time"
+      // merged/closed totals), so every row must be folded into the durable orb_outcome_rollups totals in the
+      // SAME transaction that deletes it. A fold and delete that could commit separately would either
+      // double-count (fold landed, delete didn't, next run re-folds) or under-count (delete landed, fold
+      // didn't).
+      //
+      // BOUNDED, like every other table here. An earlier revision deleted the whole aged cohort in one
+      // unbatched statement, arguing the daily volume is small -- true today, and exactly the argument that
+      // ages badly (one fleet-wide backfill and it is a multi-million-row statement). Instead each slice
+      // picks a TIMESTAMP boundary and both statements share the identical `occurred_at` predicate, so they
+      // provably see the same rows without either depending on rowid/ctid (which pg-dialect rewrites to a
+      // PHYSICAL location -- the #9470 hazard) or on row-value tuple syntax (uneven SQLite support).
+      //
+      // Slice boundary = the `occurred_at` of the row at OFFSET batchSize-1 among expired rows, ascending,
+      // taken INCLUSIVELY (`<=`). Inclusive is what makes progress guaranteed even when many rows share one
+      // timestamp: an exclusive `<` boundary against a run of ties selects zero rows and spins forever (there
+      // is a dedicated regression test for exactly that). The slice is therefore batchSize rows plus any ties
+      // at the boundary, and the final slice (no row at that offset) takes the remainder with the real cutoff.
+      let deleted = 0;
+      for (;;) {
+        const boundaryRow = await env.DB.prepare(
+          `SELECT ${rule.column} AS boundary FROM ${rule.table} WHERE ${retentionWhere(rule)} ORDER BY ${rule.column} LIMIT 1 OFFSET ${batchSize - 1}`,
+        )
+          .bind(cutoff)
+          .first<{ boundary: string }>();
+        // `== null` deliberately: D1 drivers disagree on .first() returning null vs undefined for no-row.
+        const sliceIsFinal = boundaryRow == null;
+        const sliceWhere = sliceIsFinal ? `${rule.column} < ?1` : `${rule.column} <= ?1`;
+        const sliceBound = sliceIsFinal ? cutoff : boundaryRow.boundary;
+        const batchResults = await env.DB.batch([
+          // Fold EXACTLY the population getOrbGlobalStats counts: registered installations only, and only
+          // outcomes whose PR never published a review surface (those are already counted by the own ledger).
+          // Rows failing either filter are deleted WITHOUT folding -- the live query never counted them, so
+          // folding them would make the public total jump on prune day. Keyed per lowercased account_login so
+          // the stats query's excludeAccount de-dup keeps working against the rollup after the raw rows are gone.
+          env.DB.prepare(
+            `INSERT INTO orb_outcome_rollups (account_login, merged, closed, total, updated_at)
+             SELECT LOWER(COALESCE(i.account_login, '')) AS account_login,
+                    SUM(CASE WHEN o.outcome = 'merged' THEN 1 ELSE 0 END) AS merged,
+                    SUM(CASE WHEN o.outcome = 'closed' THEN 1 ELSE 0 END) AS closed,
+                    COUNT(*) AS total,
+                    ?2 AS updated_at
+             FROM orb_pr_outcomes o
+             JOIN orb_github_installations i ON i.installation_id = o.installation_id AND i.registered = 1
+             LEFT JOIN audit_events ae
+               ON ae.target_key = o.repository_full_name || '#' || o.pr_number
+               AND ae.event_type = 'github_app.pr_public_surface_published'
+             WHERE o.${sliceWhere} AND ae.id IS NULL
+             GROUP BY LOWER(COALESCE(i.account_login, ''))
+             ON CONFLICT(account_login) DO UPDATE SET
+               merged = orb_outcome_rollups.merged + excluded.merged,
+               closed = orb_outcome_rollups.closed + excluded.closed,
+               total = orb_outcome_rollups.total + excluded.total,
+               updated_at = excluded.updated_at`,
+          ).bind(sliceBound, nowIso()),
+          env.DB.prepare(`DELETE FROM ${rule.table} WHERE ${sliceWhere}`).bind(sliceBound),
+        ]);
+        /* v8 ignore next 2 -- defensive: batch() returns exactly one result per statement on both backends, so
+         * the `?.`/`?? 0` arms only satisfy the driver types; a missing meta degrades the COUNT, never the prune. */
+        const changes = Number(batchResults[1]?.meta?.changes ?? 0);
+        deleted += changes;
+        // The final slice always ends the loop; a non-final slice that somehow removed nothing would too,
+        // so a driver anomaly can never spin here.
+        if (sliceIsFinal || changes === 0 || deleted >= maxPerTable) break;
+      }
+      results.push({ table: rule.table, column: rule.column, cutoff, deleted });
+      continue;
+    }
+
     let deleted = 0;
     // Batched delete by a real indexable PK (see RETENTION_PK_COLUMN) ordered by the retention column, so
     // each statement is bounded AND the inner SELECT is an index range scan on Postgres, not a ctid-keyed

src/openapi/spec.ts

@@ -1641,8 +1641,8 @@ export function buildOpenApiSpec() {
     path: "/v1/public/decision-ledger/verify",
     summary: "Verify a window of the hash-chained decision ledger (resumable via afterSeq)",
     responses: {
-      200: { description: "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing." },
-      409: { description: "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | short_tail (a record exists past the verified tip with no chain entry)" },
+      200: { description: "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." },
+      409: { description: "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | missing_record | content_mismatch | short_tail (a record newer than the verified tip has no chain entry — the truncated-tail signature) | unchained_record (an INTERIOR record has no chain entry — the failed-append signature). Records younger than the 5-minute append grace window are not reported: the record insert and its chain append are two writes moments apart, and a verify landing between them is not evidence of tampering." },
     },
   });
   registry.registerPath({

src/orb/outcomes.ts

@@ -64,10 +64,19 @@ export async function getOrbGlobalStats(env: Env, opts: { excludeAccount?: strin
   // excludeAccount de-dups an account already counted by another source. "" = include all.
   const exclude = (opts.excludeAccount ?? "").toLowerCase();
   let row: { merged: number | null; closed: number | null; total: number | null } | null;
+  let rollup: { merged: number | null; closed: number | null; total: number | null } | null;
   // #8879: guard ONLY the query. A D1 error degrades to zeros, mirroring computeFleetAnalytics's try/catch
   // (src/orb/analytics.ts) so a failure on this join drops just the orb aggregate instead of 503-ing the entire
   // /v1/public/stats payload (accuracyTrend/reuseRateTrend/reviewVolumeTrend/rulePrecision) via the route catch.
   try {
+    // #9474: rows older than the 90-day retention window are folded into orb_outcome_rollups by the prune
+    // (atomically with their deletion -- see pruneExpiredRecords) precisely so THIS cumulative total never
+    // shrinks. Rollup rows key on the lowercased account_login, so the same excludeAccount de-dup applies.
+    rollup = await env.DB.prepare(
+      `SELECT SUM(merged) AS merged, SUM(closed) AS closed, SUM(total) AS total FROM orb_outcome_rollups WHERE (?1 = '' OR account_login <> ?1)`,
+    )
+      .bind(exclude)
+      .first<{ merged: number | null; closed: number | null; total: number | null }>();
     row = await env.DB.prepare(
       `SELECT
        SUM(CASE WHEN o.outcome = 'merged' THEN 1 ELSE 0 END) AS merged,
@@ -88,5 +97,10 @@ export async function getOrbGlobalStats(env: Env, opts: { excludeAccount?: strin
   }
   /* v8 ignore next -- an aggregate query always returns exactly one row; this guards the nullable .first() type only */
   if (!row) return { merged: 0, closed: 0, total: 0 };
-  return { merged: row.merged ?? 0, closed: row.closed ?? 0, total: row.total ?? 0 };
+  // The SUM() arms are genuinely reachable NULLs, not defensive: an aggregate over zero rows returns NULL.
+  return {
+    merged: (row.merged ?? 0) + (rollup?.merged ?? 0),
+    closed: (row.closed ?? 0) + (rollup?.closed ?? 0),
+    total: (row.total ?? 0) + (rollup?.total ?? 0),
+  };
 }