test(db): reach every declared source-health state in every run

docs/contributing/oracle-coverage.md

@@ -3403,6 +3403,28 @@ does not generate a hook input without `cwd`, a worktree outside the launch
 checkout, or a worktree branch that has no copy of the script. In that last
 case the launch checkout's rule lints the worktree.
 
+`packages/db/tests/declared-reach-oracle.property.test.ts`
+(`oracle.declared-reach`) owns when `createOracleReach` (ORC-015) checks reach,
+and what fixed examples prove. By default the check runs in an ordinary run and
+in a run with only `TANSTACK_DB_ORACLE_SEED`, which reseeds the whole campaign
+at its full budget. Only a path replay, which runs one recorded history, skips
+it. A property can run fixed examples first in every run, through
+`oraclePropertyOptions(…, { examples })`. The examples come on top of the base
+run budget, so they never displace a random history. With `fixedHistories`,
+the check also fails for each declared state that the examples did not enter.
+The `source-health.history`, `source-health.on-demand`, and
+`source-health.lazy-demand` properties run each reach prefix, and each driver
+option that a declared state needs, as examples, so their reach does not depend
+on the seed (#2287). The oracle generates ledgers over two to four states, up to
+six histories, minimums up to three, and up to four fixed histories, under the
+three run kinds. It also drives `runOracleCampaign` in
+`packages/db/tests/oracle-campaign.ts`, whose seed-only run of its own replay
+prefix checks reach in the same way. A declared state of another property that
+only random histories reach can still miss by chance at the normal budget.
+Those properties have not been measured. Destination: #2289. Needed witness:
+per property, the declared states that no fixed example enters, and how often a
+normal-budget run misses each one.
+
 `packages/db/tests/oracle-replay.test.ts` checks every replay manifest owner:
 from its cwd, the command's files exist, its target is the owner, and a Vitest
 config applies.

packages/db/tests/declared-reach-oracle.property.test.ts

@@ -0,0 +1,355 @@
+/**
+ * Declared reach (ORC-015) cannot miss by chance where a property fixes it.
+ *
+ * A generated property declares the states its laws depend on, and
+ * `createOracleReach` fails a run in which too few histories entered one.
+ * Two rules decide when that check runs and what it proves (#2287):
+ *
+ * - The check runs in an ordinary run and in a run with only
+ *   `TANSTACK_DB_ORACLE_SEED`: a seed alone reseeds the whole campaign at its
+ *   full budget. Only a path replay, which runs one recorded history, skips
+ *   it.
+ * - A property can run fixed examples first in every run. With
+ *   `fixedHistories: n`, the check also fails for each declared state that
+ *   the first n histories, the examples, did not enter. Such a state's reach
+ *   would depend on the seed. The examples come on top of the base run
+ *   budget, so they never displace a random history, and a path replay still
+ *   runs one history.
+ *
+ * `runOracleCampaign` follows the first rule for its own replay prefix: a
+ * seed-only run of its campaign checks reach, and only a path replay skips it.
+ *
+ * The model below states both rules over a generated ledger: a run kind, a
+ * minimum, a number of fixed histories, and the states each history enters.
+ * The drivers run `createOracleReach`, `runOracleCampaign`, and
+ * `oracleRandomParameters` under that run kind's environment, and the checks
+ * compare counts, the failure, and the states the failure names with the
+ * model.
+ */
+import { fc, test as fcTest } from '@fast-check/vitest'
+import { afterAll, expect, vi } from 'vitest'
+import { createOracleReach } from './oracle-reach.js'
+import { runOracleCampaign } from './oracle-campaign.js'
+import {
+  oraclePropertyOptions,
+  oracleRandomParameters,
+} from './oracle-config.js'
+import { declareOracle } from './oracle-declaration.js'
+
+const property = `oracle.declared-reach`
+
+type RunKind = `ordinary` | `seed` | `path`
+type Ledger = {
+  run: RunKind
+  states: ReadonlyArray<string>
+  minimum: number
+  fixedHistories: number
+  // The indices of the states each history enters, with repeats.
+  histories: ReadonlyArray<ReadonlyArray<number>>
+}
+type Verdict = {
+  counts: ReadonlyMap<string, number>
+  belowMinimum: ReadonlyMap<string, number>
+  unfixed: ReadonlyArray<string>
+}
+
+// The grammar: two to four states, and up to six histories over them.
+const ledgerArbitrary: fc.Arbitrary<Ledger> = fc
+  .integer({ min: 2, max: 4 })
+  .chain((stateCount) =>
+    fc.record({
+      run: fc.constantFrom<RunKind>(`ordinary`, `seed`, `path`),
+      states: fc.constant(
+        Array.from({ length: stateCount }, (_, index) => `state ${index}`),
+      ),
+      minimum: fc.integer({ min: 1, max: 3 }),
+      fixedHistories: fc.integer({ min: 0, max: 4 }),
+      histories: fc.array(
+        fc.array(fc.integer({ min: 0, max: stateCount - 1 }), {
+          maxLength: 4,
+        }),
+        { maxLength: 6 },
+      ),
+    }),
+  )
+
+// The model. A history counts once per state it enters. A path replay checks
+// nothing. Otherwise a state fails below the minimum, and, when fixed
+// histories are declared, a state the first ones did not enter fails too.
+function expectedVerdict(ledger: Ledger): Verdict | `skipped` {
+  if (ledger.run === `path`) return `skipped`
+  const entered = ledger.histories.map(
+    (history) => new Set(history.map((index) => ledger.states[index]!)),
+  )
+  const counts = new Map(
+    ledger.states.map((state) => [
+      state,
+      entered.filter((history) => history.has(state)).length,
+    ]),
+  )
+  const fixed = entered.slice(0, ledger.fixedHistories)
+  return {
+    counts,
+    belowMinimum: new Map(
+      [...counts].filter(([, count]) => count < ledger.minimum),
+    ),
+    unfixed:
+      ledger.fixedHistories === 0
+        ? []
+        : ledger.states.filter((state) =>
+            fixed.every((history) => !history.has(state)),
+          ),
+  }
+}
+
+// The environment each run kind sets.
+function runEnvironment(run: RunKind): Record<string, string> {
+  if (run === `ordinary`) return {}
+  if (run === `seed`) return { TANSTACK_DB_ORACLE_SEED: `1` }
+  return {
+    TANSTACK_DB_ORACLE_SEED: `1`,
+    TANSTACK_DB_ORACLE_PATH: `0`,
+    TANSTACK_DB_ORACLE_PROPERTY: property,
+  }
+}
+
+function withEnvironment<T>(variables: Record<string, string>, run: () => T) {
+  const names = [
+    `TANSTACK_DB_ORACLE_SEED`,
+    `TANSTACK_DB_ORACLE_PATH`,
+    `TANSTACK_DB_ORACLE_PROPERTY`,
+  ]
+  const saved = names.map((name) => [name, process.env[name]] as const)
+  for (const name of names) delete process.env[name]
+  Object.assign(process.env, variables)
+  try {
+    return run()
+  } finally {
+    for (const [name, value] of saved) {
+      if (value === undefined) delete process.env[name]
+      else process.env[name] = value
+    }
+  }
+}
+
+// The driver: one ledger through createOracleReach.
+function observeLedger(ledger: Ledger) {
+  const reach = withEnvironment(runEnvironment(ledger.run), () =>
+    createOracleReach(`ledger`, ledger.states, {
+      minimum: ledger.minimum,
+      fixedHistories: ledger.fixedHistories,
+    }),
+  )
+  for (const history of ledger.histories) {
+    reach.beginHistory()
+    for (const index of history) reach.reach(ledger.states[index]!)
+  }
+  let failure: string | undefined
+  try {
+    reach.assertReached()
+  } catch (error) {
+    failure = (error as Error).message
+  }
+  return { counts: new Map(reach.counts()), failure }
+}
+
+// The states a failure names: `name (count)` below the minimum, and a list of
+// names for the fixed histories.
+function namedStates(failure: string) {
+  const below = /not enforced there: (.*)$/m.exec(failure)?.[1]
+  const unfixed = /reach depends on the seed: (.*)$/m.exec(failure)?.[1]
+  return {
+    belowMinimum: new Map(
+      (below?.split(`, `) ?? []).map((entry) => {
+        const match = /^(.*) \((\d+)\)$/.exec(entry)!
+        return [match[1]!, Number(match[2])] as const
+      }),
+    ),
+    unfixed: unfixed?.split(`, `) ?? [],
+  }
+}
+
+// The check: the observed ledger refines the model.
+function checkLedger(ledger: Ledger) {
+  const expected = expectedVerdict(ledger)
+  const observed = observeLedger(ledger)
+  if (expected === `skipped`) {
+    expect(observed.failure, `a path replay checks no reach`).toBeUndefined()
+    return expected
+  }
+  expect(observed.counts, `histories per state`).toEqual(expected.counts)
+  const fails = expected.belowMinimum.size > 0 || expected.unfixed.length > 0
+  expect(observed.failure !== undefined, `the reach check fails`).toBe(fails)
+  if (observed.failure !== undefined) {
+    const named = namedStates(observed.failure)
+    expect(named.belowMinimum, `states below the minimum`).toEqual(
+      expected.belowMinimum,
+    )
+    expect(named.unfixed, `states the fixed histories missed`).toEqual(
+      expected.unfixed,
+    )
+  }
+  return expected
+}
+
+// The campaign runner's replay prefix, and the environment each run kind sets
+// for it.
+const campaignPrefix = `TANSTACK_DB_REACH_LEDGER`
+function campaignEnvironment(run: RunKind): Record<string, string> {
+  if (run === `ordinary`) return {}
+  if (run === `seed`) return { [`${campaignPrefix}_SEED`]: `1` }
+  return { [`${campaignPrefix}_SEED`]: `1`, [`${campaignPrefix}_PATH`]: `0` }
+}
+
+// The driver: one ledger through each campaign that runOracleCampaign runs.
+async function observeCampaign(ledger: Ledger) {
+  const names = [`${campaignPrefix}_SEED`, `${campaignPrefix}_PATH`]
+  const saved = names.map((name) => [name, process.env[name]] as const)
+  for (const name of names) delete process.env[name]
+  Object.assign(process.env, campaignEnvironment(ledger.run))
+  // The runner reports each campaign's counts on the console.
+  const info = vi.spyOn(console, `info`).mockImplementation(() => {})
+  try {
+    await runOracleCampaign(campaignPrefix, ledger.states, (reach) => {
+      for (const history of ledger.histories) {
+        reach.beginHistory()
+        for (const index of history) reach.reach(ledger.states[index]!)
+      }
+    })
+    return undefined
+  } catch (error) {
+    return (error as Error).message
+  } finally {
+    info.mockRestore()
+    for (const [name, value] of saved) {
+      if (value === undefined) delete process.env[name]
+      else process.env[name] = value
+    }
+  }
+}
+
+// The check: a campaign's reach refines the model with a minimum of one and
+// no fixed histories, the campaign runner's own reach.
+async function checkCampaign(ledger: Ledger) {
+  const expected = expectedVerdict({
+    ...ledger,
+    minimum: 1,
+    fixedHistories: 0,
+  })
+  const failure = await observeCampaign(ledger)
+  if (expected === `skipped`) {
+    expect(
+      failure,
+      `a path replay of a campaign checks no reach`,
+    ).toBeUndefined()
+    return
+  }
+  expect(failure !== undefined, `the campaign's reach check fails`).toBe(
+    expected.belowMinimum.size > 0,
+  )
+  if (failure !== undefined)
+    expect(namedStates(failure).belowMinimum, `states below one`).toEqual(
+      expected.belowMinimum,
+    )
+}
+
+// The run budget: the examples come on top of the base runs, except in a
+// path replay of this property, which runs one history.
+function expectedNumRuns(run: RunKind, baseRuns: number, examples: number) {
+  return run === `path` ? 1 : baseRuns + examples
+}
+
+function checkRunParameters(
+  run: RunKind,
+  baseRuns: number,
+  examples: ReadonlyArray<number>,
+) {
+  const parameters = oracleRandomParameters(
+    baseRuns,
+    run === `ordinary`
+      ? undefined
+      : run === `seed`
+        ? { replaySeed: 1, replayPath: undefined, replayProperty: undefined }
+        : { replaySeed: 1, replayPath: `0`, replayProperty: property },
+    property,
+    examples,
+  )
+  expect(parameters.numRuns, `runs`).toBe(
+    expectedNumRuns(run, baseRuns, examples.length),
+  )
+  expect(parameters.examples, `examples`).toEqual(examples)
+}
+
+declareOracle({
+  check: [checkLedger, checkCampaign, checkRunParameters],
+  model: [expectedVerdict, expectedNumRuns],
+  grammar: [ledgerArbitrary],
+})
+
+const reachStates = [
+  `a seed-only run fails below the minimum`,
+  `a path replay with a state below the minimum`,
+  `a fixed history miss that a later history enters`,
+  `a state below a minimum above one`,
+  `a ledger that passes`,
+] as const
+// One fixed ledger enters each reach state, first in every run, so this
+// property's own reach cannot miss by chance.
+function fixedLedger(
+  run: RunKind,
+  minimum: number,
+  fixedHistories: number,
+  histories: Ledger[`histories`],
+): [Ledger, number, Array<number>] {
+  return [
+    { run, states: [`state 0`, `state 1`], minimum, fixedHistories, histories },
+    60,
+    [1],
+  ]
+}
+const ledgerExamples = [
+  fixedLedger(`seed`, 1, 0, []),
+  fixedLedger(`path`, 1, 0, []),
+  fixedLedger(`ordinary`, 1, 1, [[0], [1]]),
+  fixedLedger(`ordinary`, 2, 0, [[0, 1]]),
+  fixedLedger(`ordinary`, 1, 0, [[0, 1]]),
+]
+const ledgerReach = createOracleReach(`oracle.declared-reach`, reachStates, {
+  fixedHistories: ledgerExamples.length,
+})
+let ledgersRan = false
+afterAll(() => {
+  if (ledgersRan) ledgerReach.assertReached()
+})
+
+fcTest.prop(
+  [
+    ledgerArbitrary,
+    fc.integer({ min: 1, max: 200 }),
+    fc.array(fc.integer(), { maxLength: 5 }),
+  ],
+  oraclePropertyOptions(200, property, { examples: ledgerExamples }),
+)(
+  `reach ledgers refine the declared reach model`,
+  async (ledger, baseRuns, examples) => {
+    ledgersRan = true
+    ledgerReach.beginHistory()
+    const verdict = checkLedger(ledger)
+    await checkCampaign(ledger)
+    checkRunParameters(ledger.run, baseRuns, examples)
+    if (verdict === `skipped`) {
+      const ordinary = expectedVerdict({ ...ledger, run: `ordinary` })
+      if (ordinary !== `skipped` && ordinary.belowMinimum.size > 0)
+        ledgerReach.reach(`a path replay with a state below the minimum`)
+      return
+    }
+    if (ledger.run === `seed` && verdict.belowMinimum.size > 0)
+      ledgerReach.reach(`a seed-only run fails below the minimum`)
+    if (verdict.unfixed.some((state) => (verdict.counts.get(state) ?? 0) > 0))
+      ledgerReach.reach(`a fixed history miss that a later history enters`)
+    if (ledger.minimum > 1 && verdict.belowMinimum.size > 0)
+      ledgerReach.reach(`a state below a minimum above one`)
+    if (verdict.belowMinimum.size === 0 && verdict.unfixed.length === 0)
+      ledgerReach.reach(`a ledger that passes`)
+  },
+)