diff --git a/.changeset/fix-correlated-callback-metadata.md b/.changeset/fix-correlated-callback-metadata.md new file mode 100644 index 0000000000..7ba161e160 --- /dev/null +++ b/.changeset/fix-correlated-callback-metadata.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Fix reserved metadata before `fn.select()`, `fn.where()`, and `fn.having()` callbacks in live queries, Effects, and correlated includes. Preserve genuine source metadata, authored inline values, and selected output when compiled queries share a cache. Preserve every ordered window and joined-source publication dependency when one source feeds multiple compiled views or nested queries. diff --git a/.changeset/retain-loading-sibling-replay.md b/.changeset/retain-loading-sibling-replay.md new file mode 100644 index 0000000000..64368eee0d --- /dev/null +++ b/.changeset/retain-loading-sibling-replay.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Keep the last complete live-query result when a new include demand fails while a sibling subscription is still replaying after truncate. A successful sibling load cannot publish an incomplete replacement. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 6e5ba7182e..1d10868379 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -796,27 +796,66 @@ stay ready/empty, and a new interval gets a new handle. This assumes no facade-local optimistic mutations. Every shape declares active empty readiness and nonempty child receiving cuts. Six lifetime and four inline-value hostile controls reject wrong observations. See [chunk 5](oracle-reviews/2026-10-28-pr-2171-oracle-chunks.md). -Functional callback-input observations now have a separate receiving boundary -in `virtual-row-fields-input-oracle.test.ts`, sharing source metadata judgment -with the existing root/Effect owners. Its finite 33 authoring descriptions cross -seven QueryRef/join/union consumers through root and correlated inline-child -compilation (462 histories), plus two direct-source controls. Each compares -immutable callback captures and their ordinary derived output at initial -readiness, held mutation, settlement and remote next use. Complete before/after -source facts permit retraction evaluation; startup LEFT joins may expose an -absent side, while established outputs require both typed-key rows. Reserved -metadata is judged by presence/readable value, with ownership flags diagnostic; -ordinary own undefined remains distinct from absence. Another 66 histories keep -nested inline array/materialized authored values, and 33 reject compiled -Collection inputs before callbacks. Per-history/path/cut reach is mandatory even -under unrelated replay seeds; unentered calibration-only runs stand aside. -Unchanged production exposes 203 correlated-child violations: new-shape fields -leak and source-spread collisions spoof genuine metadata. Root receivers and -nested inline preservation pass. All 2,153 declared states reach; 17 controls -and independent seeded/unentered/partial-selection checks calibrate this owner. -See [chunk 17](oracle-reviews/2026-10-28-pr-2171-oracle-chunks.md) for exact -fingerprints and independent evidence. Production repair is deferred under the -[RC oracle challenge](oracle-reviews/2026-10-28-pr-2171-oracle-challenge.md). +Functional callback-input observations have a separate receiving boundary in +`virtual-row-fields-input-oracle.test.ts`, sharing the existing source metadata +judgment. The maintainer extended O12 on 2026-10-30 from fn.select to fn.where +and fn.having while preserving authored inline output. Thirty-three authoring +descriptions cross seven QueryRef/join/union paths, five callback forms +(select, where, having, where-select, having-select), and root, wrapped child, +direct child and real Effect receivers. Flat union children require a named +QueryRef for include correlation; direct flat union children remain inadmissible. +The finite grammar has 4,290 receiving histories and 20 direct-source controls. + +Each receiver checks immutable callback captures and complete derived or +predicate-filtered output at initial readiness, held mutation, settlement and +remote next use. Effects also check exit/reentry, every delivered value and +previousValue, continuity in both callback channels and silence after disposal. +Before/after source facts permit retraction evaluation. Startup LEFT joins may +expose an absent side; after source exit the unmatched peer remains a result. +Reserved fields use presence/readable value, with ownership diagnostic. +Ordinary own undefined remains distinct from absence. Another 132 histories +retain inline array/materialized authored values through direct and QueryRef +paths; 66 reject compiled Collection inputs before callbacks. All 20,720 +callback and 594 inline/rejection states reach once. + +Another 396 histories use exported `compileQuery`, the same exact QueryIR and +D2 input streams, both compilation orders, shared/separate caches, and FROM, +joined-source and union-branch placement. At initial graph drain, all 1,188 +states compare callback captures, complete derived output and raw authored +output. Raw output uses a shared-cache/separate-cache differential, while both +executions still undergo the independent callback/derived-output judgment. +The union optimizer clones branch IR: counters establish original-key reuse +for FROM/JOIN, while union checks propagation/transparency without claiming +that cache hit. This receiver supplies no Effect-root metadata attribution law. + +The owner has 4,923 tests. Before callback production edits, it rejected 1,102 +metadata input comparisons on the reviewed implementation; 3,821 tests passed. +All 22,502 callback, inline and compiler states reached. The repaired owner +passes. Final mutants reject a WHERE-only repair (551 comparisons), enabling +normalization for every predicate pipeline (696 authored-output comparisons), +and dropping the selected-row cache mode (26 comparisons). Earlier path-specific +FROM/JOIN/union normalization mutants each failed 112 compiler comparisons. +The real Effect receiver additionally rejects 203 metadata comparisons on the +actual PR base. That receiving evidence does not retroactively supply the +historical selection repair's initial pre-edit gate. + +The ordered-work owner, `ordered-work-oracle.property.test.ts`, adds 144 finite +histories through `ordered-compiler-cache-driver.ts`. Two compilation orders, +shared/separate caches, FROM/JOIN/union paths, two projection forms, two order +directions and three window moves exercise actual source Collections and the +real OrderedSourceLoader. The independent installed-window model checks signed +output bags at initial, moved and returned cuts. Original registration overwrite +fails 48 histories; the repaired full 424-test owner passes. All 432 new states +reach once. This receiver claims the controlled compiler/loader boundary, not +public Collection window-promise integration. Its 280 existing histories retain +their Collection/Effect receiving evidence. Separate metadata-mode pipelines +remain a permitted representation; no one-pipeline performance promise follows. +The [runtime repair record](oracle-reviews/callback-metadata.md) retains the +commands, source bindings, lossless nine-item audit, calibrations and limits. +Historical instrument acceptance remains in +[chunk 17](oracle-reviews/2026-10-28-pr-2171-oracle-chunks.md). +The separate static receiving-type failures in the +[deferred record](oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md) remain open. This extension does not establish arbitrary descriptors/prototypes, greater recursive depth, same-source overlapping transactions, other dynamic join kinds, truncate, provider scheduling, framework receivers or construction @@ -1108,6 +1147,23 @@ that fails one receiver's current request while comparing another receiver's status at the same checkpoint. This is a stated observation limit, not a retirement of the per-demand error-ownership law. +The replay error-identity comparison keeps subscription ownership separate +from the source-wide publication barrier. CI seed `1926301329`, path +`251:2:1:4:2:2`, exposed a false failure: an include replay rejected while a +join replay still loaded, and the comparison required the join's earlier +failure instead. Adapter request data now identifies each load's subscription. +Only failures and covering replacements within that subscription compete for +its replay error. A 96-history finite grammar crosses root/marker replay, +oldest/newest failure order, four join forms, both directions and three window +widths. It measures 52 histories with multiple failures of one subscription +and six with sibling failures; both require at least five. The old source-wide +comparison and a production mutant that returns the last replay failure each +fail the intended identity assertion. Publication, request-error, retirement, +readiness and convergence comparisons and campaign budgets remain. This is a +controlled Collection/live-query witness, with no provider or framework claim. +Evidence and the oracle self-review are in +[the callback metadata record](oracle-reviews/callback-metadata.md#replay-error-identity-ci-follow-up-2026-10-30). + The ordered replay owner distinguishes replay renewal from an ordinary lazy demand created during another subscription's replay. Its adapter records the leases held at truncate, then matches subscription identity and independently @@ -5939,6 +5995,27 @@ path from a later run. Until then, this entry cannot tell an operator regression from an older defect. A later failure of that campaign must record its seed here before anyone calls it pre-existing. +### Ordered joined dependency composition, October 30 + +October 30 receiving extension: the same ordered-work owner now checks joined +dependency composition across direct, wrapped QueryRef and two-filter nested +ordered plans. Its 128 finite histories cross actual Collection and Effect +receivers, two directions, matched/anti-joined membership, widths one/two, +joined answer order and which of two joined sources vetoes a new head row. +Each history checks settled initial rows, then every recorded publication and +public read at held, partial and settled cuts. All 256 declared held/settled +states reach their minimum once. Unchanged `8464ec977` fails 24 publication +comparisons (12 per receiver); the repair passes all 128 histories. Overwrite, +last-only and first-only dependency mutants fail 24, eight and eight intended +comparisons respectively. The original 424 owner tests and their budgets stay. +This is a controlled adapter witness through real core receivers: one inserted +head row, two joined sources, one QueryRef level and windows of one/two rows. +Ordinary progressive Effect startup callbacks remain outside this new atomicity +claim. No provider/framework, arbitrary nesting, rejection, cleanup or public +window-promise integration claim follows. See the versioned +[joined dependency review](oracle-reviews/callback-metadata.md#ordered-joined-dependency-review-2026-10-30) +for the pre-edit gate, experiments, source binding and loss audit. + ### Ordered signalling-refetch work scope, October 27 Owner: `packages/db/tests/query/ordered-work-oracle.property.test.ts`. The @@ -6230,6 +6307,17 @@ bundle erasure, Typedoc generation, tooling GREEN and restored-old-tooling mutants remain required receiving checks. Generated docs remain CI-owned. +### Replay successful-retirement ownership, October 30 + +The load-subset replay refinement owner scopes successful replacement to the +same subscription as the failure. Its 96-history failure/success matrix reaches +four sibling-success coverage histories and calibrates that another subscription's +failure survives the account action. The original account fails that comparison; +all 44 normal owner tests pass after repair, with existing budgets intact. +No production change or new provider/framework claim follows. See the +[versioned retirement review](oracle-reviews/callback-metadata.md#replay-retirement-scope-review-2026-10-30). + + ### Distinct collision review extension (2026-10-30) The existing distinct incrementalization owner now also generates four unequal @@ -6304,3 +6392,37 @@ useful extension: the hash identity oracle owns those semantic distinctions, but its helper-level evidence does not by itself exercise distinct storage. The separate Index structural-collision obligation above remains open. See the appended [review evidence](oracle-reviews/2026-10-30-distinct-collisions.md). + +## Loading sibling replay and final catch-up reach (2026-10-30) + +The primary load-subset replay owner now constructs 48 finite histories where +an include with no demand at truncate starts under its sibling's loading replay. +It measures held failure (44 histories), failure retirement (33), and later own +coverage (12), through the existing public retention and recovery comparisons. +The original guard fails the quiet retention check. Subscription-scoped verdict +and successful-coverage calibrations remain in that primary owner. + +The browser multi-tab Electric owner preserves its stop prefix and arbitrary +tail and constructs an abandoned reservation immediately before final catch-up. +This removes dependence on sampling that boundary without changing its model, +80-run budgets, comparisons or reach minima. + +See [the versioned experiment record](oracle-reviews/2026-10-30-release-replay-retention.md) +for authority, RED, reach, mutant, grammar controls, historical gate limits and +receiving-path limits. This is core Collection publication evidence and a Node +SQLite/coordinator receiving witness with controlled Electric streams. It does +not claim Effect, provider, framework, deeper include, or browser-process closure. + + +### Subscription-scoped request-error assertion (2026-10-30) + +The primary load-subset replay owner now retains subscription identity in its +later-request and successful-coverage account. A sibling requesting overlapping +source keys cannot exempt the failed demand's public health error. Its existing +96-history sibling-success grammar measures the actual owed-error assertion in +four histories, alongside four successful-sibling-coverage histories. A targeted +omitted-error observation passes the former source-wide account and fails the +scoped health comparison. All existing comparisons, histories, budgets and +reach minima remain. The controlled core, one-level include and peer limits +remain those of this owner; no provider or framework claim follows. See the +[versioned repair record](oracle-reviews/2026-10-30-release-replay-retention.md#subscription-scoped-request-error-checker-repair). diff --git a/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md b/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md index c92603d698..704a97a3e6 100644 --- a/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md +++ b/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md @@ -19,6 +19,13 @@ No follow-up PR is claimed until one is actually created. callbacks. The approved callback law remains normative. The accepted runtime owner has 377 passes / 203 intended REDs and all 2,153 cells; the bounded concrete lane has the same seven failures and one compatible pass on main. + The runtime follow-up restores this owner and adds direct child receivers. + It reproduces 348 REDs before repair and passes all 979 tests afterward, with + all 3,510 states reached. This includes a low-level shared compiler-cache + receiver whose 132 histories reproduced 26 failures before its cache repair. + Inline QueryRef histories also reject an overbroad + first repair before the corrected production edit. See [the runtime repair](callback-metadata.md). + This does not close the separate static metadata receiving-type failures below. - P11/P12: raw undefined becomes null and selected empty records disappear. Their root/QueryRef runtime owner retains 18 REDs, correlated receiver 72 REDs, and source-public type receiver 280 REDs. Every declared receiving cell and diff --git a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md new file mode 100644 index 0000000000..06e78d2e28 --- /dev/null +++ b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md @@ -0,0 +1,217 @@ +# Release replay retention and generated final catch-up + +The release workflow for `e75f8455a78bc33c43376288218e9912dc2a3f4c` +failed before publication. Its two failures have different causes. +The DB property exposed a production retention defect. The browser property +passed its behavior comparisons but missed a required state. Neither failure +establishes a regression in the callback-metadata size refactor: the DB +counterexample also fails on the exact release source. + +## Law and the first divergent observation + +Authority: live-query `ARCHITECTURE.md`, **Replay participants and failure**, +and normative law 9. A query's subscriptions to one source share publication. +Every acquisition started under that replay participates. A failed held demand +keeps the last complete result. Sibling success cannot decide its verdict. +Demand retirement or successful own replacement are the neighboring controls. + +The original `replay.ordered-failed-replay` counterexample uses seed +`-533188721`, path `428:3:3:6`. Its commands hold loads, truncate markers, +settle the two oldest loads successfully, fail the newest load, then remove +marker 1. The scenario has descending order, offset 1, limit 2, no ties or +peer, an anti-join and an inline include over on-demand markers. At quiet +loading, the expected retained snapshot is empty; production publishes row 1. + +The adapter receipts distinguish production failure from demand retirement. +The include held no demand at truncate. It later acquires marker key 1 while +the sibling join replay is loading. That include acquisition fails and remains +held and un-aborted. Its sibling later succeeds. Only cleanup unloads the +failed acquisition, after the mismatching observation. The old guard looked +only for a failed sibling replay. The repair also recognizes a loading replay +that still withholds publication. Ordinary failures outside replay retain their +existing request-error behavior. + +## Primary owner and experiment order + +`load-subset-replay-refinement-oracle.test.ts` remains the primary owner. +The change retains the subscription-scoped verdict and successful-coverage +corrections previously reviewed on PR 2312. Those corrections concern the +model, not a new production promise: shared publication does not merge replay +verdicts. Their two finite calibrations remain in the same owner. + +Before the runtime edit, a new constructive finite grammar produced the +intended RED on unchanged main. The failing assertion reports one measured new +sibling failure and the same empty-snapshot/row-1 mismatch. The first reach +recorder incorrectly excluded retired historical acquisitions; the corrected +pre-edit run measures acquisitions actually held at truncate. Both logs remain. + +The final grammar has 48 histories: two order directions, three widths, one or +two successful pre-failure settlements, and four later actions (none, marker +change, rank move, or another marker replay). Every history uses the existing +model, driver, retention comparison after each command, quiet checkpoint, and +final authoritative recovery. No expected row set was added to the histories. + +The initial repair trial passed all retention comparisons but failed the newly +declared own-coverage reach. The final recorder observes same-subscription key +coverage, rather than predicate object identity: a later demand may construct +new request data. The grammar also includes an explicit later replay control. +This post-trial reach work does not retroactively establish a pre-edit recovery +reach gate. The original retention RED and the receiving path were obtained +before production changed. + +Final measured reach: 44 histories with a first-demand failure under a loading +sibling; 33 with retirement of such failed demand; 12 with later successful +coverage by its own subscription. The three states belong to this constructive +grammar. Adding them to the shared result type initially made the unconstrained +campaign demand them by accident; the explicit owner assignment corrects that +without removing any pre-existing reach obligation. + +The original-guard mutant fails at the intended quiet retention comparison, +with the same one measured new sibling failure. It was restored in a `finally` +block and the repaired source hash was checked. Setup errors are recorded +separately and are not RED or mutant rejection evidence. + +## Browser grammar repair + +`multi-tab-electric-oracle.test.ts` owns the existing five browser laws. +The failed 80-history random campaign missed **reset-abandoned source recovered +during final catch-up**. Its fixed campaign reached that state four times. +A previous stop/peer prefix constructs other required states, but this state +still depended on an unconstrained 8–24-step tail. + +The repair keeps that prefix, tail, 80-run budgets, model, comparisons and reach +minimum. A suffix creates a fresh handle, recovers and catches up every tab, +delivers a new change to one tab without committing it, then resets its +reservation. The existing final delivery must observe the abandoned commit's +failure before restarting and checking convergence. Two/three tabs, source tab, +key and recovery method vary. Removing the partial delivery or reset loses the +premise; committing before final catch-up moves the failure to a different cut. +The original named final-delivery replays remain unchanged. + +## Review and limits + +This is an author self-review. The earlier independent diagnosis supplied the +receipt trace; it is not an independent review of the final implementation. + +- ORC-001/002/003/005/009: accepted architecture law, existing independent + row/retention model, literate grammar and real Collection sync boundary; + subscription identities and requested keys are adapter receipts. +- ORC-004/013/015: finite direction/width/settlement/change grammar reconstructs + the reported witness and adjacent controls. Absent replay is the ordinary + failure neighbor; obsolete or retired requests cannot count as held failures. + Browser suffix ablations lose its stated premise. Reach is measured. +- ORC-006/010: original guard rejected at the retention assertion; original + failure, initial reach-only failures and setup errors remain distinguishable. + Existing fixture cleanup remains in place. +- ORC-007: existing fixed/random properties and direct seed/path interface are + unchanged. The new 48-cell grammar is finite enumeration, not a random + property. Browser fixed/random campaigns retain the same 80-run grammar. +- ORC-008: no new semantic model state. New receipt fields serve reach only. + Retained subscription distinction is required because sibling success and + own success permit different verdicts. +- ORC-011: no new shared semantic fault hypothesis. Independent adapter trace + rejected the alternative retirement explanation of this failure. +- ORC-012/014/016: this record states the bounded claim and the unchanged + controlled premises. Histories use the shared comparison, with no unit-test + expected answers. + +Core evidence is limited to live-query Collection publication with one ordered +source, a one-level inline include, and the controlled core sync adapter. This +change does not claim Effect failure disposal, provider delivery, framework +scheduling, arbitrary joins or deeper recursion. Those paths have separate +owners; the modified builder failure path serves live-query Collections. +Browser evidence uses real SQLite, BroadcastChannel and coordinator code in one +Node process with controlled Electric streams. It does not establish installed +SDK/HTTP, real browser multi-process timing, on-demand row contents or existing +known-gap closure. No laws, assertions, run budgets or reach minima are weakened. + +## Final commands, results and source binding + +Local runs use saved configs under `/private/tmp/callback-metadata/release-replay-repair` to resolve the exact worktree source with cached dependencies. Every command sets `pnpm_config_verify_deps_before_run=false`; Vitest uses one worker. + +| Check | Command or selector | Result | +| --- | --- | --- | +| Pre-edit primary RED | `pnpm exec vitest run --config /vitest.config.mjs -t 'keeps new include failures behind a loading sibling replay'` | Intended quiet retention mismatch; measured first-demand sibling failure 1. | +| Final finite grammar | Same command | PASS, 48 histories, failure/retirement/own-recovery reach 44/33/12. | +| Original guard mutant | Same command with only the old guard restored | Intended quiet retention mismatch, measured failure 1. Repaired source restored and hash verified. | +| Full primary owner | `pnpm exec vitest run --config /vitest.config.mjs` | 43 passed, two accidental new-reach obligations failed. After explicit ownership correction, both affected fixed/random campaigns passed. All 45 tests pass across those runs. | +| Core neighbors | Saved `neighbors.config.mjs` | 21 tests in three files pass, covering replay refinement, failure boundaries and request errors. | +| Browser fixed/random campaigns | `pnpm exec vitest run --config /browser.config.mjs -t 'keeps reserved commits lawful'` | Both pass. Final catch-up recovery reached 80/80 histories in each campaign. | +| Browser final-delivery controls | Same config with the two existing final-delivery selectors | Both existing shared-model replays pass. | +| Focused TypeScript | `pnpm exec tsc --noEmit -p /db-types.json` and `browser-types.json` | Both pass. | + +The browser campaign retains 80 runs for each seed mode. Its expected injected stream errors are diagnostics, not comparison failures. The first two browser setup failures, the initial reach-only failures, and the intermediate TypeScript literal-inference error remain separate from RED/GREEN evidence. + +The following SHA256 fingerprints bind the main-based executable evidence to PR 2318 commit `0e5a5234b9fe4a26244a0e52d3862916a7f4ce7e`. The callback branch port has separate source fingerprints and results in [its evidence record](callback-metadata.md#release-blocking-loading-sibling-replay-port-2026-10-30). + +| File | SHA256 | +| --- | --- | +| `packages/db/src/query/live/collection-config-builder.ts` | `bb1b62aaf00aed1dc5eaa5e537671d92bb99def7a53a6e951ef38f73b00f5088` | +| `packages/db/src/collection/subscription.ts` | `bd171bdc97b38a79ebbc81d7fb605267787308559ad4c0646dff1e072a43798a` | +| `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts` | `e5ce4a77fe8d16f4b0d26f4c71fdadb53b2779c8e7e8313a7eb32d2de3ebdd38` | +| `packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts` | `67c16d284522417f1a77e0a36efcc552b718f858ea8ec00771f001f001160782` | + +Focused ESLint reports zero errors and two existing `no-shadow` warnings. Prettier, whitespace, and `pnpm lint:changed` pass. No run budget decreased or disappeared. + +## Subscription-scoped request-error checker repair + +CodeRabbit review 5719714297 covered repair head +`0e5a5234b9fe4a26244a0e52d3862916a7f4ce7e`. Its one behavioral +finding identified a false-green request-error classification. Architecture +law 16 and the glossary's request-error rule already scope a lazy demand's +error to that demand. A sibling subscription asking for or loading the same +source keys cannot decide it. Shared publication does not merge verdicts. +The existing complete authoritative source-recovery control still decides the +held demands together; an individual sibling verdict does not supply it. + +The old account stored successful keys by source alone. Both its later-request +and later-coverage exemptions omitted subscription identity. The repair keeps +that identity in the recorded successful acquisition and requires it in both +exemptions. This is an oracle enforcement repair. Production is unchanged. + +The existing 96-history sibling-success grammar varies direction, join form, +width, replay source and settlement order. A receipt probe found four histories +with eight observation cuts where the old account exempted an error the scoped +account owed. Production listed the exact injected error at all eight cuts. +The cut concerns a later sibling request; the experiment did not establish a +later successful sibling-coverage cut with an otherwise owed request error. +An initial reach counter for that stronger premise correctly failed at zero. +It remains in the scratch evidence. The final declared reach names the actual +sibling-request error comparison and reaches four histories. The existing +successful-sibling-coverage reach also remains four. + +Calibration supplies a wrong public observation to the same health comparison: +omit the injected error only at those distinguishing cuts. The old account +accepts it. The scoped account rejects it at `after settle: request error`, +with expected true and actual false. A broader temporary production mutant +that suppressed every ready-query replay error already failed the old checker; +it is not evidence for this newly repaired gap. Both temporary sources were +restored. The real production owner passes all 45 tests, including fixed and +random campaigns, with the stronger classifier. The reported seeds and paths +from issues 2294 and 2315 each pass the direct replay interface as well. These +are bounded replay results, not closure of either whole issue or provider +adapter conformance from issue 2118. + +ORC review (author self-review): 001/002/003 use the accepted demand-scoped +law, independently evaluated request keys and explanatory local prose. +004/013 retain all 96 legal histories and their direction/join/width/source/ +settlement neighbors; collapsing subscription identity is the wrong rule. +005/006/015 exercise the real Collection sync path and public health error +identity after each command, measure four distinguishing histories and reject +the targeted omission. 007 retains all fixed/random campaigns, budgets and +seed/path replays. 008/009 retain subscription identity because equal source +keys can have different demand verdicts. 010 retains primary mismatch and +cleanup separation. 011 has no new shared production/model semantic-fault +hypothesis: the defect was an exemption before the assertion. 012 is this +versioned bounded record. 014 keeps the controlled core boundary and makes no +provider/framework handoff claim. 016 adds no unit expectation or standalone +fixture. No law, comparison, budget or prior reach minimum was weakened. + +The checker evidence is stored under +`/private/tmp/callback-metadata/release-replay-repair/review-5719714297`. +Commands use the saved `vitest.config.mjs` for the full primary owner and +`-t 'retains sibling replay failures through successful coverage'` for the +finite calibration, with one worker and dependency verification disabled. +The amended primary owner SHA256 is `a2ab454ff5a77aa47bf57cbd590b67a2a5602d3dd6897af051fb1819b9482ce7` on both branches. The earlier +table binds the original repair evidence. Its runtime and browser hashes remain +unchanged; callback runtime hashes remain bound by its separate port record. diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md new file mode 100644 index 0000000000..1bec01c769 --- /dev/null +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -0,0 +1,1051 @@ +# Callback metadata runtime repair + +The accepted callback-input law applies before user JavaScript derives a value. +Direct source returns and structured whole-source spreads expose genuine +`$key`, `$origin`, `$hasPendingWrites` and `$collectionId`. New projected shapes, +including functional spreads, expose none of these reserved fields to downstream +callbacks. Inline include values retain their authored shape. Compiled +Collection-valued callback inputs reject before user code runs. + +Authority: the maintainer's O12 decision from 2026-10-28 and the +[architecture's callback-input boundary](../../../packages/db/src/query/live/ARCHITECTURE.md#route-context-transport). +The original repair enforced the selection-input law. The maintainer extended +it to functional predicates on 2026-10-30, as recorded in the nine-finding +review below. Earlier evidence in this record describes the historical repair. +The [coverage owner](../oracle-coverage.md) retains the separate static type gaps. + +## Independent owner and receiving histories + +The restored owner consists of the +[model and grammar](../../../packages/db/tests/query/virtual-row-fields-input-oracle.ts), +[production driver](../../../packages/db/tests/query/virtual-row-fields-input-driver.ts) +and [runner and controls](../../../packages/db/tests/query/virtual-row-fields-input-oracle.test.ts). +It derives expectations from authored source facts and projection syntax. +It shares the source metadata law with the existing publication owner. +It imports no compiler metadata classifier for its expected result. + +Thirty-three legal shape/collision descriptions cover five projection forms, +no collision, each reserved name, all names, and both authored pending booleans. +Direct source returns admit no authored override. Seven QueryRef/join/union +paths run at root and through a wrapped correlated child: 231 histories each. +Five paths also run directly as the correlated child: 165 histories. Flat union +children have no named source reference for include correlation, so their +supported forms use the wrapper or union-query-ref path. They remain covered. +Three direct-source controls, 132 inline array/materialized histories, 66 compiled +Collection rejections and 19 checker/recorder controls form 847 tests. Inline +values cross direct and QueryRef paths. The QueryRef path correlates each parent +with one typed-key child, so it preserves authored child fields without a child +functional callback. A later low-level compiler receiver adds 132 histories, +bringing the owner to 979 tests. Its separate initial graph-drain cut is +described below. + +Each Collection receiving history observes initial readiness, a held optimistic update, +settlement without confirmation, and a later remote update. Source rows have +numeric `1` and string `"1"` keys and distinct labels. Each callback copies its +input immediately, then returns that capture as ordinary output. The comparison +checks every callback against coherent before/after source facts and checks the +complete derived output at each established cut. Retractions may read the old +row. Startup LEFT joins may expose an absent joined side. Neither exception +permits mixed old/new fields or missing peers at the established output cut. + +Reserved fields promise readable presence/value, without a descriptor layout. +Ordinary own undefined and authored nested inline properties retain exact shape. +Failures accumulate through later cuts. Cleanup preserves the primary mismatch +and attempts every resource release. The model retains source facts, not a +second compiler or query queue. + +## RED before production edits + +The production baseline was `c351fc04d7f70dc59bf92ecbed310f899b9511cd`. +The three original owner files were restored from accepted source tree +`db38b4e009146af53e46aeeef3206269f5968af8` before any compiler edit. +The original 580 tests reproduced 203 correlated-child failures and 377 passes. +All 2,153 original reach states reached once. + +The direct child receiver was then added on unchanged production. The complete +746-test owner produced 348 comparison failures and 398 passes. Root controls +and inline authored values passed. Wrapped children contributed 203 failures; +direct children contributed 145. All 2,520 callback and 297 inline/rejection +states reached once, including later cuts after the first violation. + +Two representative distinguishing traces are retained by the executable grammar: + +- `inline-child/query-ref/functional-projection/$collectionId/false`, initial: + the projected callback input exposed authored `$collectionId: "spoof-$collectionId"`. + The model requires that field to be absent. The ordinary derived capture + preserved this earlier violation through final publication. +- `inline-child-direct/joined-source/structured-spread/all/true`, initial: + the callback input exposed authored pending `true`, origin `local`, and spoofed + key/Collection ID. The genuine source was remote, pending `false`, with numeric + key `1` or string key `"1"` and Collection ID `callback-first`. + +The first setup attempt lacked `fake-indexeddb/auto` because the new worktree +had no package dependency link. It ran zero tests and supplies no RED evidence. +Early direct flat-union fixtures failed correlation admission. The legal grammar +uses a named QueryRef for those unions. Neither setup failure counts as a defect. +No comparison, reach minimum or run budget was weakened to obtain GREEN. + +## Repair and discriminators + +The first repair enabled source/projection metadata mode at every recursive +QueryRef and joined source. A neighboring-path probe found that this was too +broad: an inline child with a QueryRef, but no functional callback, lost its +authored reserved fields. The existing inline grammar used direct children only. + +Before another production edit, the inline grammar added QueryRef consumption +across all shapes, collisions and materialization forms: 99 more histories. +The first repair then failed 58 shared callback/output comparisons and passed +787 tests. For example, `query-ref/array/structured-projection/$key/false` +lost the child's authored `$key: "spoof-$key"` at the initial callback capture. +The unchanged baseline compiler passes all these inline histories and still +fails the original 348 callback comparisons: 497 passes out of 845 tests. +These runs distinguish both accepted laws before the corrected repair. + +The corrected repair requests source/projection metadata mode when the local +query has a functional selection, or when the enclosing query already requires +Collection publication mode. It passes that mode into FROM and joined-source +compilation. An inline-only QueryRef retains its authored selected shape. +Existing mode parameters and propagation remain. That repair changes two +call sites. The later cache repair below separates their compiled results by +mode. Neither adds a metadata registry, traversal or public API. + +The pre-review head passed all 845 tests. Every one of the 2,520 callback and 594 inline/rejection +states reaches once. Seven neighboring suites pass 1,883 tests: root metadata, +functional projection/input rejection, route-context transport, public container +copying, include query shapes and subquery user values. Focused no-emit TypeScript, +ESLint and changed-file oracle lint pass. The normal package pnpm configuration +also passes all 845 owner tests after integrating #2301 +(`bfd1da1a7e752d66fd667adfd3cdf54f60732b22`). No local build or installation ran. + +Three wrong repairs ran sequentially in copied source trees before the +independent review. They did not change the reviewed worktree. These counts use +the 845-test owner at `a30256cfb`, before the two added recorder calibrations: + +| Mutant | Intended wrong design | Result at the shared comparison | +| --- | --- | --- | +| QueryRef-only | Keep the old joined-source mode | 58 failures, 787 passes | +| Sanitize inline | Sanitize every final child instead of preserving authored inline values | 116 failures, 729 passes | +| Unconditional sources | First repair, ignoring whether a callback consumes the rows | 58 failures, 787 passes | + +Every failure is a `MetadataCallbackOracleFailure` at the intended input/output +comparison. No mutant timed out or failed setup. The original 348-failure RED +rejects the original defect. Direct receivers distinguish an unrepaired joined +source, and the expanded inline paths reject both blanket sanitization repairs. + +## Prep review and loss audit + +The initial read-only review and simplification passes found stale opening +counts and required coverage/architecture updates. They missed the inline QueryRef +regression found by the later neighboring-path probe. That gap required the +grammar extension and corrected repair described above. + +A proposed shared-query cache conflict was tested through actual Effects and +live-query Collections. Each receiver reused one projected query through a +QueryRef, then through both union branch orders. All six probes passed on both repairs: the +nested callback saw no `$collectionId`. The direct Effect branch retained its +authored reserved output, which is separate from downstream callback input. +The initial Collection-union fixture supplied conflicting custom public keys; +using the compiler's union keys corrected the fixture. Those initial failures +establish no product violation. + +The probe constructs the following source and nested consumer before selecting +QueryRef-only, `unionAll(projected, consumed)`, or its reverse: + +```ts +import { createCollection, localOnlyCollectionOptions, Query } from '@tanstack/db' + +const source = createCollection(localOnlyCollectionOptions({ + initialData: [{ id: 1 }], + getKey: (row: { id: number }) => row.id, +})) +const captures: Array<{ present: boolean; value: unknown }> = [] +const projected = new Query().from({ source }).select(({ source }) => ({ + id: source.id, + $collectionId: 'spoof', +})) +const consumed = new Query().from({ nested: projected }).fn.select(({ nested }) => { + captures.push({ present: '$collectionId' in nested, value: nested.$collectionId }) + return { id: nested.id, observed: nested.$collectionId } +}) +``` + +Each of the six runs entered the callback once and captured +`{ present: false, value: undefined }`. This refutes that concrete cache +hypothesis. It is no universal cache proof or generated Effect lifecycle claim. +The source-level review distinguished final inline/Effect values from callback +input. Its finite probes supply useful cross-path evidence. The reviewers did +not independently rerun the full owner or mutation campaign. + +The initial four review items and the new inline finding reconcile as three +fixed items, one duplicate count correction and one refuted cache hypothesis. +The new finding retains both its first-repair RED and corrected GREEN evidence. +Initial reviewers provided useful documentation and cache analysis, but their +missed inline path limits confidence in their completeness. The final independent +review and simplification pass found no new defect or useful code reduction. +The reviewer also ran two inline LEFT-join controls without a child callback: +one selected QueryRef on FROM, the other on the joined side. Both selected the +whole projected child into `toArray()` and retained authored +`$collectionId: "authored"` after preload. These are supplemental initial-cut +controls, not generated lifecycle coverage. Their repeated documentation notes +link to the original count item, giving seven raw items: three fixed, one +refuted and three duplicates. Neither review supplies a whole-library verdict. + +## Independent oracle review after PR creation + +Four subagents used the repository oracle-review skill on +`a30256cfbaf99138f274db224362325c37ea1da4`. They independently challenged laws, +grammar/reach, driver/comparisons, and mutation/process evidence. + +The law review confirmed a historical process gap: the first pre-edit receiving +inventory omitted inline-only QueryRef preservation. That path was added after +the first production repair. The later baseline run cannot satisfy the initial +gate retrospectively. The corrected production edit did have the expanded RED +before its edit. This record preserves that distinction and claims no complete +initial-gate compliance. + +The driver review found a recorder blind spot. `Array.map` copied inline array +entries but dropped named own fields. An ordinary `Object.assign(children, +{ unexpected: "spoof" })`, or its `$collectionId` counterpart, remained publicly +readable but disappeared from capture. Both shared judgments accepted it. +This was a confirmed test-integrity defect, not a reproduced production defect. + +Two new calibration cases ran before the recorder edit. Both failed because +both judgments returned false for rejection. Capture now preserves named own +string fields recursively as well as array entries. `Object.defineProperty` +avoids setter traps. The expected model and both judgments are unchanged. +All 847 owner tests pass: the existing 845 plus two recorder calibrations. +All 2,520 callback and 594 inline/rejection states still reach once. Restoring +the old recorder is the demonstrated wrong design: both new calibrations fail. +The reviewer inspected the fix and found that it closes the reported gap. + +The mutant reviewer independently omitted only the FROM callback-mode change, +leaving joined-source handling intact. On copied sources at the reviewed head, +290 comparisons fail and 555 tests pass. The first distinguishing history is +`inline-child/query-ref/structured-projection/$key/false`, initial: the callback +reads authored `$key: "spoof-$key"` where the model requires absence. Every +failure is `MetadataCallbackOracleFailure`; an earlier configuration EPERM ran +no tests and supplies no evidence. This supplements the three mutants above. + +The grammar review independently reconciled all 845 original tests and the +per-receiver reach matrix. Flattened child output does not prove parent-child +route placement. It retains every child's metadata capture, typed key and +occurrence count; those reviews found no hidden metadata violation from that +flattening. Parent routing remains owned by the separate include oracles. + +The review accounts for two findings: one repaired recorder gap and one +historical gate miss preserved above. No current production defect emerged. +The source-only law and grammar reviews supplied useful independent reasoning. +The driver review supplied an executable false-green witness and a reviewed +repair. The mutant review supplied an additional independent discriminator. +These results support the finite runtime boundary, not a whole-library verdict. + +CI also rejected this record's query example because its source was untyped. +The example now declares its typed source and capture array. Its focused +snippet typecheck passes without a baseline exclusion. That mechanical docs +repair is separate from the two oracle-review findings. + +The full local snippet scan reports 12 failures in unchanged examples and one +stale baseline entry under the reused dependency tree. The earlier CI scan +reported only this record's example as a new failure. No dependency install or +baseline exclusion was used to mask that local environment difference. Full +final-head snippet verification then remained for CI. CI on `9bd623cd5` +subsequently passed its full scan: 805 examples, zero new failures, with the +existing baseline unchanged. + +## Shared compiler-cache receiving boundary + +CodeRabbit review `5715750975` on `9bd623cd5` identified a missing cache premise. +The exported `compileQuery` accepts raw QueryIR and an optional cache. Existing +`compiler/subquery-caching.test.ts` controls reuse that cache across calls on +one graph. Cache reuse must preserve the accepted callback-input law and the +authored selected output that separate compilations produce. + +Reusing a QueryBuilder alone did not reach this premise. Ten Effect/Collection +builder probes and two single-root raw-IR probes passed; placement clones or +optimizer lineage gave them distinct effective raw queries. Two compilations +on the same graph/input streams and exact selected QueryIR do reach it: + +| Compilation order with one cache | Wrong public observation | Required observation | +| --- | --- | --- | +| Authored selected root, then downstream callback root | The callback reads `$key: "spoof-$key"` from a structured projection | Reserved fields are absent before the callback | +| Downstream callback root, then authored selected root | The selected output loses authored `$key` | The selected output retains `"spoof-$key"` | + +Separate-cache neighbors pass both orders. The defect was reproduced again +without compiler instrumentation. Earlier scratch shape/literal fixture +mistakes and a diagnostic setup error supply no defect evidence. + +Before production edits, the primary owner added 132 histories: the existing +33 source/projection/collision descriptions, both compilation orders, and +shared/separate caches. Actual public source rows retain numeric `1` and string +`"1"` peers. The existing independent model judges immutable callback captures, +ordinary derived output and raw authored selected output. Signed-bag recording +retains duplicate contributions and retractions. No expected answer comes from +the separate-cache implementation. All three comparisons run before failure. + +Unchanged `9bd623cd5` production gives 953 passes and 26 +`MetadataCallbackOracleFailure` comparisons out of 979 tests. All original +847 tests pass. All 396 new compiler states and all 3,114 prior states reach +once. The driver checks initial graph drain only: its authored-output judgment +is a low-level compiler boundary, not a new real inline include or optimistic +lifecycle witness. Two independent source reviews found no additional defect +in the grammar, recorder, comparisons or proposed cache key. + +The repair retains a result per publication mode under each raw QueryIR key. +Uncorrelated-only admission and same-mode reuse remain. The graph's shared +value-identity scope remains the same. All 979 owner tests pass afterward, +with every one of the 3,510 reach states reached. The original cache is the +hostile wrong design rejected by the 26 REDs. It can be reconstructed by +restoring `QueryCache = WeakMap`, lookup with +`cache.get(rawQuery)`, and storage with `cache.set(rawQuery, compilationResult)`. +No model rule or comparison changes between RED and GREEN. + +Normal package configuration passes all 3,170 targeted tests across ten files, +including this owner, the eight compiler-cache controls, the Effect metadata +owner and the seven neighboring suites named above. Focused no-emit TypeScript, +ESLint, formatting and whitespace checks pass. A temporary RED configuration +only redirects Vite's cache around the reused dependency symlink permissions. +Its jsdom setup, source aliases and shared comparisons are unchanged. The +package-configuration setup failure ran no tests and is excluded. + +This closes the recorded cache-mode violation within the 132 histories and +their initial cut. Arbitrary raw-IR shapes, independent graphs, later input +updates and construction failure need separate evidence. The historical +initial receiving-inventory miss remains documented above. + +## Rubric accounting and limits + +ORC-001–005: accepted authority, independent source facts, literate layers, +finite legal grammar, real callback captures and exact derived output cuts. +Removing receiver, callback path, inline path, cache sharing/order, collision, pending boolean or checkpoint loses a +named source/projection/inline distinction. Both populated union positions and +both typed keys remain represented. Direct-source controls distinguish a +blanket stripping rule. The rejected flat-union form bounds admission. + +ORC-006: 19 hostile observation/recorder controls, the three recorded production +mutants, the independent FROM-only mutant and the original wrong cache. +ORC-007: exhaustive finite enumeration, so random campaigns and seed replay do +not apply. At the pre-review head, a valid unrelated `utils.1` seed/path run passes all 845 tests with +all reach intact. A calibration-only selection passes 17 controls without +entering a receiver. Selecting only the three direct-source controls fails the +missing-matrix reach assertion. An omitted replay property was a configuration +error with zero tests, not product evidence. ORC-008–009: +source facts distinguish coherent retractions; callback input is distinct from +published row, with that abstraction stated beside the model. ORC-010: all-cut +failure reports and failure-preserving cleanup. ORC-011: builder probes missed +the exact raw-query/cache identity premise. The low-level receiver now reaches +that premise in both orders and rejects the wrong cache. ORC-012: this +versioned record and source fingerprints below. +ORC-013: single/all/no collisions, typed peers, source/new/inline forms and the +three mutants distinguish plausible shortcut repairs. ORC-014: the declared +boundary is core runtime compilation, with no real-provider/framework timing +claim. ORC-015: every named callback, inline/rejection and compiler state reaches once. +ORC-016: all histories and controls use the shared independent judgments. + +The runtime evidence is finite: flat inline children, at most two QueryRefs, +one LEFT join, two typed-key peers, one held successful mutation, and four cuts. +The separate low-level cache receiver uses one graph and initial graph drain. +Arbitrary descriptors/prototypes, deeper recursion, same-key overlapping writes, +other join kinds, truncate, framework/provider scheduling and construction-failure +cleanup remain outside it. The deferred static metadata receivers and literal +value/type defects remain open in the [retained record](2026-10-29-pr-2171-deferred-oracles.md). +This runtime repair does not claim complete static or P11/P12 closure. + +## Reproduction and reviewed sources + +From the repository root, run this targeted owner without a build or install: + +```sh +TANSTACK_DB_ORACLE_STATISTICS=1 pnpm_config_verify_deps_before_run=false \ + pnpm --dir packages/db exec vitest run --maxWorkers=2 \ + --coverage.enabled=false --typecheck.enabled=false \ + tests/query/virtual-row-fields-input-oracle.test.ts +``` + +The source fingerprints identify the reviewed implementation independently of +later prose edits. The parent baseline and exact legal grammar reproduce the +original failures without temporary log files. + +| Source | SHA256 | +| --- | --- | +| `packages/db/src/query/compiler/index.ts` | `a00931f51624d31b4ee749a741fdea30289344a63309b5e2b48b44ee00b913c2` | +| `packages/db/src/query/compiler/types.ts` | `7a8244358fea67f64dac84215adcc079aba4188977b2eb628673632b15c40be6` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.ts` | `d0d526479dda63a01906bcb377675120386e02fe26720ebc7b2c3b7b6e57e658` | +| `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `b24f421e3f5126078d7103ba6ba6a9d7bdd90251c1b4b69ca55e57c020e10292` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `22f075260e1ff8be769409a12b39ce51436c19936d47324d9f866260ab40cfcc` | + +## Nine-finding review and predicate extension (2026-10-30) + +Reviewed production: `5ba869f0fc8398243a45a923828b717f0050f40a`. +The implementation and executable evidence below supersede the historical +selection-only counts and source bindings above. The raw nine-item review and +append-only working ledger remain in +`/private/tmp/callback-metadata/nine-issue-review-2312/`. + +### Law, experiment and inference + +The original O12 decision named fn.select. Later architecture prose claimed +functional callbacks generally. The maintainer resolved that conflict on +2026-10-30: extend the input law to select, where and having, while preserving +authored inline output. This is an explicit contract extension. The earlier +selection owner enforced its narrower law but could not generate predicates. + +Direct source rows and structured whole-row source spreads retain genuine +reserved metadata at each source-input callback. A projection with its own +shape exposes none. Callback evaluation must not rewrite the authored result +that an inline child exports. The callback owner derives both views from +independent source facts and projection syntax. Functional predicates actually +derive membership from their captured input, so a spoof can change output. +Predicate-with-selection controls distinguish placement from callback syntax. +Grouped HAVING evaluates its selected group context rather than a local source +namespace; this extension does not revise group attribution or arbitrary +user-created selected-context semantics. + +The extended finite owner has 4,923 tests: 4,290 receiving histories, 20 direct +source controls, 198 inline/rejection histories, 396 compiler histories and 19 +checker controls. The receiving paths are root Collection, wrapped inline +child, direct inline child and real Effect. Each checks initial readiness, +held mutation, settlement and remote next use. Effects also check exit, +reentry, both event channels, every value/previousValue, event continuity and +post-disposal silence. LEFT-join absence has its own model observation. + +The final pre-edit predicate gate produced 1,102 MetadataCallbackOracleFailure +comparisons and 3,821 passes. All 20,720 callback, 594 inline/rejection and 1,188 +compiler states reached once. Initial recorder attempts incorrectly framed +absence and the flat union's `$selected` context. Those setup/evaluation errors +are retained separately and do not supply production RED. Reach now records +callback/host-cut entry independently of output correctness; comparisons remain +strict. Production had only mechanical flag/cache edits and the separately +gated ordered repair before this predicate RED. The callback boundary itself +remained unrepaired. + +The production repair carries a normalized callback view with each row +occurrence beside its authored selected value. FROM, JOIN and union adapters +transport it. Functional callbacks consume that view, then the existing walker +removes private state. Output also removes private state, including for raw +Effect roots. No global object-to-provenance registry merges distinct row +occurrences. Inline output keeps authored values. Selected-row normalization +has an explicit name; its two cache slots replace the per-query boolean Map. +The existing single source-mode expression serves FROM and JOIN. + +The compiler receiver now checks raw selected output by a shared-cache versus +separate-cache relation. It no longer borrows an inline-include attribution +rule for an exported compiler root. Both executions independently check +callback input and derived output, so their agreement cannot excuse a shared +spoof. Its 396 histories cross both orders and all three consuming paths. +Original-key counters distinguish actual FROM/JOIN reuse from optimizer-cloned +union branches. Union coverage establishes propagation/transparency, not an +original-key cache hit. + +A second law requires every ordered window to receive the source rows it +needs. The original source-plan assignment orphaned an earlier top-K: widening +its window from one to three left output `[1]`, while reversed compilation +order produced `[1,2,3]`. The probe recorded two registrations and 15 graph +operators. These are measured work facts, not a universal pipeline budget. +The ordered-work owner now crosses compilation order, cache sharing, FROM/JOIN/ +union placement, projection form, order direction and three window moves: +144 histories and 432 initial/moved/returned states. Its independent existing +installed-window model checks signed output bags. Before the order-by repair, +48 histories failed and 376 tests passed; all old 280 histories passed. +Registration now combines live window bounds and data-needed judgments. It +requires the full source when orders differ. Individual top-K windows and +setters remain independent. All 424 ordered owner tests pass. + +This ordered receiver uses a real source Collection and OrderedSourceLoader +with a controlled subscription/provider seam. It claims the compiler/loader +cut, not public window-promise integration. The existing owner separately +retains its real Collection/Effect lifecycle coverage. + +### Enforcement evidence + +Scratch root: `/private/tmp/callback-metadata/nine-issue-review-2312`. +Commands run with `pnpm_config_verify_deps_before_run=false`, without installs, +builds or lockfile changes, and with at most two Vitest workers. + +| Experiment | Result and evidence | +| --- | --- | +| Unchanged callback production, extended predicate owner | 1,102 failed / 3,821 passed, `predicate-owner-red-corrected.{json,log}` | +| Repaired callback owner | 4,923 passed, `boundary-attempt3.{json,log}` | +| Actual PR base `bfd1da1a7e752d66fd667adfd3cdf54f60732b22` with new Effect receiver | 203 Effect comparisons failed, `pr-base-owner.{json,log}` | +| Original ordered source registration | 48 failed / 376 passed, `ordered-owner-red.{json,log}` | +| Repaired ordered owner | 424 passed, `ordered-owner-green.{json,log}` | +| Normal package configuration, eight neighboring suites | 5,912 passed, `final-package-green.{json,log}` | +| WHERE-only callback repair | 551 HAVING comparisons failed, `final-where-only.{json,log}` | +| Normalize every predicate's source pipeline | 696 authored-output comparisons failed, `final-sanitize-all-predicates.{json,log}` | +| Lose selected-row cache mode | 26 compiler comparisons failed, `final-raw-only-cache.{json,log}` | +| Restore source-plan overwrite | 48 ordered comparisons failed, `final-overwrite-window.{json,log}` | +| Earlier FROM/JOIN/union mode omissions | 112 compiler comparisons failed per path, `mutant-summary.json` | + +The final mutants ran against copied final sources and final tests. They failed +at the intended comparisons, rather than during setup. Their source edits and +full failure traces remain in `run-final-mutants.py`, `final-mutant-baseline/` +and `final-mutant-summary.json`. The original predicate RED also rejects the +reviewed implementation as a hostile wrong design. No comparison, reach +minimum or run budget was lowered. + +### Lossless item audit + +| ID | Technical verdict | Action and durable value | +| --- | --- | --- | +| EXT2312-01 | Confirmed predicate input spoof for WHERE and HAVING, including real Effects and inline children. The original select-only authority required the maintainer's resolved extension. | fixed-now: occurrence callback view plus callback-kind grammar, membership and authored-output checks. | +| EXT2312-02 | Confirmed broad prose exceeded the implemented and generated select-only promise. No predicate comparison existed to enforce it. | fixed-now: approved law, executable grammar and current architecture/coverage claims agree. Historical initial-gate gaps remain explicit. | +| EXT2312-03 | Confirmed real Effect receiver was missing. Its actual PR-base run rejects 203 metadata comparisons. | fixed-now: shared-model Effect lifecycle receiver and corrected changeset. New evidence does not retroactively repair the original pre-edit process. | +| EXT2312-04 | Confirmed absolute raw-root judgment lacked an independent attribution authority. | fixed-now: differential raw-output law, retaining independent input/output checks in both executions. | +| EXT2312-05 | Confirmed two mode pipelines and the orphaned ordered plan, with an actual underfilled result. Pipeline duplication alone violates no stated resource law. | fixed-now: primary ordered owner plus live aggregate source plan. Separate top-Ks remain permitted. No unsupported one-pipeline or speed claim. | +| EXT2312-06 | Confirmed duplicate expression at FROM/JOIN. | fixed-now: compute the expression once. Path-specific mutation evidence preserves the reason. | +| EXT2312-07 | Confirmed ambiguous publication name. The cited branches normalize selected metadata rather than performing Collection publication. | fixed-now: rename to sanitizeSelectedMetadata and separate callback view transport. No independent root-only behavior defect follows from the old name. | +| EXT2312-08 | Confirmed a Map represented only two slots. No heap or bundle measurement supported the claimed magnitude. | fixed-now: typed two-slot tuple. Claim only the removed Map structure, not measured bytes or speed. | +| EXT2312-09 | Confirmed FROM-only cache grammar omitted JOIN and union. | fixed-now: three-path grammar and path-specific mutant sensitivity. Preserve the union optimizer-cloning limit. | + +All nine raw items have separate IDs, claims, proposed repairs, evidence and +dispositions in the append-only ledger. Total: nine fixed-now, zero deferred, +zero unresolved product decisions. The ordered-loader counterexample is the +confirmed consequence of item 5, rather than an extra invented raw item. +The initial repair attempt exposed private callback-view state in raw Effect +output; final result cleanup removes it, and Effect continuity rejects that +attempt. This collateral attempt is retained as implementation feedback. + +The reviewer earns a strong hire recommendation. The predicate probes identified +a real placement-dependent leak, and source analysis predicted the ordered-plan +failure that execution confirmed. The differential and path-dimension proposals +improve test integrity. The flag and allocation comments are useful cleanup, +with lower severity than the correctness findings. The review did not establish +a performance budget or an original-key union cache hit. Those qualifications +limit the claims without discarding the findings. + +### Guide audit and limits + +ORC-001–003: the approved law, limits, independent model, grammar, real receivers +and comparisons remain beside their executable definitions. ORC-004: existing +33 legal descriptions retain all single/all/no collisions and typed peers; +callback-kind ablation loses predicates, selection controls lose placement +independence, and path ablation loses a distinct transport boundary. Compiled +Collection inputs and direct-return overrides remain excluded by admission. +ORC-005: each receiving cut and every delivered Effect value is compared. +ORC-006: the recorded original and final mutants fail their intended judgments. +ORC-007: callback/cache additions exhaust finite grammars, so seed campaigns do +not apply; existing ordered generated campaigns keep fixed/random parity. +ORC-008–009: source facts distinguish retractions, callback snapshots differ +from Collection rows, and the controlled ordered seam has an explicit mapping. +ORC-010: all-cut reports and cleanup preserve primary failures. ORC-011: +cache transparency uses two executions while independent input truth prevents +a shared false green. ORC-012: this versioned record binds the final source +hashes below. ORC-013: WHERE-only and over-sanitizing repairs demonstrate the +neighboring consequences. ORC-014: actual Collection/Effect receivers supply +core lifecycle evidence; ordered provider and framework claims stay bounded. +ORC-015: all 22,502 metadata and 432 ordered added states reach once. +ORC-016: all added histories use shared judgments, without pinned expected +answers or unit tests inside oracle files. + +The finite metadata grammar bounds recursion at two QueryRefs, joins at one +LEFT join, sources at two typed-key peers, and mutations at one held successful +update. It does not establish arbitrary descriptors/prototypes, deeper +recursion, overlapping same-key mutations, other join kinds, truncate, +provider scheduling, framework timing or construction-failure cleanup. +Grouped selected-context attribution is a separate judgment from the local +source-input predicate law. The existing aggregate and materialization owners +remain responsible for those semantics. Static metadata and P11/P12 literal/ +type gaps stay open in the retained deferred record. No universal runtime or +static closure follows from the finite evidence here. + +### Final source binding and verification + +The final package run passes all 5,912 tests across eight suites. Focused +TypeScript passes with no diagnostics. Explicit ESLint passes with zero errors +and four existing no-shadow warnings in the ordered owner. Formatting, +whitespace and changed-file oracle enforcement pass. The final four mutants +were refreshed from these exact compiler and test sources and again rejected +551, 696, 26 and 48 intended comparisons respectively. Their logs retain all +failure messages. These results supersede intermediate repair attempts. + +Run both owners under the normal package configuration: + +```sh +TANSTACK_DB_ORACLE_STATISTICS=1 pnpm_config_verify_deps_before_run=false \ + pnpm --dir packages/db exec vitest run --maxWorkers=2 \ + --coverage.enabled=false --typecheck.enabled=false \ + tests/query/virtual-row-fields-input-oracle.test.ts \ + tests/query/ordered-work-oracle.property.test.ts +``` + +The predicate RED command used the extended tests and the pre-edit compiler +binding in `predicate-gate-fingerprints.json`: + +```sh +pnpm_config_verify_deps_before_run=false pnpm exec vitest run \ + --config /private/tmp/callback-metadata/nine-issue-review-2312/owner.config.mjs \ + --reporter=json \ + --outputFile=/private/tmp/callback-metadata/nine-issue-review-2312/predicate-owner-red-corrected.json +``` + +The ordered RED used the extended ordered owner and the pre-edit binding in +`ordered-gate-fingerprints.json`, with `ordered-original.config.mjs`. The final +hostile calibrations are reproducible through `run-final-mutants.py`. The +normal eight-suite command adds the existing Effect metadata, cache-control, +functional projection/input boundary, public-container-copy and context +transport suites to the two-owner command above. No full build or dependency +installation supplied this evidence. + +These SHA-256 fingerprints bind the final behavioral sources without making +the report hash itself refer to its own contents: + +| Source | SHA-256 | +| --- | --- | +| `packages/db/src/query/compiler/index.ts` | `c566eccd683862fb3235a84e841fafca32fb8c5a43491570b2fa65a3c1874487` | +| `packages/db/src/query/compiler/joins.ts` | `1c3642f632a10b25ac784d5728e680803d657452969496a4e21b1a8213f44fcd` | +| `packages/db/src/query/compiler/order-by.ts` | `384d8a132c86265f4bb4e12ec4ed5200f062a182e35402dcd38b4d546828d0d2` | +| `packages/db/src/query/compiler/parent-routes.ts` | `5f8caf4504d70639bcd9b109da28ddb6880ebc81b1e4489e0ae8de2e7151cf42` | +| `packages/db/src/query/compiler/route-metadata.ts` | `ba0c42104288720ff221b516298d39da9e7201d68984f8101adb7df20f286cb1` | +| `packages/db/src/query/compiler/types.ts` | `e06dbdf363cc1878226692de890650680e1c84bf35f17424fd23dad09313c27d` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.ts` | `7b7c5055fe58dd701ad10db9620f516d1ab4b0127a95e17bee629310b3b5450a` | +| `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `3d157039ec6c7aead891865738346e4e52c3747fe199fca44fce0224ed4998ad` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `58ae2e54947145220a8b78a28cadc37456e3d11d507ce36846843f002d79550d` | +| `packages/db/tests/query/ordered-compiler-cache-driver.ts` | `ba7b244250754d9a5d8480847562c26907edf192c8a028973dca0de32e6c43e6` | +| `packages/db/tests/query/ordered-work-oracle.property.test.ts` | `a8e2f5884570f0659764ad0b7be7398c432b411fcf796dd007ddd3f4bb9d5e68` | + + +## Replay error identity CI follow-up (2026-10-30) + +The parent is `86510968691e0080a7484881ed7f54ba58f110af`. +This follow-up repairs an oracle scope error, with no production changes. +The earlier production and metadata-oracle fingerprints above remain intact. + +**Law and authority.** ARCHITECTURE.md, “Atomic window publication” and +“Replay participants and failure”, assign each subscription its replay's +normalized Error, the earliest failure that still decides it. Subscriptions +share a publication barrier. Their failures do not acquire a common error +order: one replay can fail while a sibling still loads. + +**Experiment.** CI failed `replay.ordered-failed-replay` with seed +`1926301329`, path `251:2:1:4:2:2`. Exact replay on unchanged production +failed the identity comparison. The trace holds loads, truncates markers, +requests window 0/1, then fails newest and oldest held loads. It uses an +ascending anti-joined window with inline includes and on-demand markers. +Adapter diagnostics identify failure load 10 with subscription 1 and the +move's rejected load 8 with subscription 2; load 9 of subscription 1 is still +pending. The source-wide comparison incorrectly made load 10 decide the +other subscription's error. This is evidence against the comparison, not a +production error or a new law decision. + +The corrected model records subscription identity from adapter request data. +Competing failures and covering replacements belong to the same subscription. +The source-wide retention model stays shared. The exact replay passes. +A 96-history finite grammar crosses two directions, four join forms, three +window widths, root/marker replay and oldest/newest failure order. Each uses +the existing real Collection/live-query driver and every shared comparison. +Declared reach records 52 histories with multiple own-subscription failures +and six with sibling failures, each with minimum five. The normal package +owner run passes all 43 tests, including unchanged fixed/random budgets. +Type checking reports no errors. + +**Calibration.** Restoring the source-wide error order fails the finite grammar +at the same marker Error identity assertion. A temporary production mutant +that returns the last replay failure instead of the first fails its root +Error identity assertion. Both reach their intended comparisons, with no type +errors. The original production source is restored byte-for-byte. +These controls distinguish an ownership correction from an unconditional +waiver of replay error order. + +**Inference and limits.** The repair preserves the accepted per-subscription +error law and freedom between sibling replay verdicts. Existing retention, +request-error, demand-retirement, readiness, window and convergence checks and +all campaign budgets remain. This bounds evidence to one query, one-level +includes, the existing finite source domain and a controlled adapter in the +real Collection/live-query path. Existing peer exclusions and receiving-provider +and framework limits remain; this is not universal replay closure. + +The evidence directory is +`/private/tmp/callback-metadata/ci-monitor/20261030T205752Z`: +`replay-red.*`, `replay-diagnostic.log`, `replay-scope-green.*`, +`replay-matrix-green.*`, `replay-package-green.*`, `source-wide-check.*`, +`last-replay-error.*`, and `replay-control-summary.json`. +The exact replay command is: + +```sh +TANSTACK_DB_ORACLE_SEED=1926301329 \ +TANSTACK_DB_ORACLE_PATH=251:2:1:4:2:2 \ +TANSTACK_DB_ORACLE_PROPERTY=replay.ordered-failed-replay \ +pnpm_config_verify_deps_before_run=false pnpm --filter @tanstack/db exec \ + vitest run tests/query/load-subset-replay-refinement-oracle.test.ts \ + -t 'keeps the last publication through a failed replay' \ + --coverage.enabled=false --maxWorkers=1 +``` + +This is an oracle self-review, not independent review. ORC conformance: + +| Requirement | Outcome | +| --- | --- | +| 001, 002 | Existing architecture authority; identity from adapter input, no production verdict helper computes expectations. | +| 003 | Contract, model distinction, finite grammar and observation remain explained beside executable code. | +| 004 | Matrix reconstructs the CI trace; removing sibling ownership reproduces the false failure. Finite domains are stated; failure of a load outside a waited replay remains outside the identity premise. | +| 005 | Real Collections and `setWindow` rejection at replay settlement; no missing source-fact hypothesis is needed for this error-identity comparison. | +| 006 | Source-wide comparison control and last-error production mutant fail intended assertions. | +| 007 | Existing fixed/random properties, direct seed/path replay and budgets remain unchanged; finite matrix is bounded enumeration. | +| 008, 009 | Subscription identity distinguishes a sibling failure from one's own; both can share the publication barrier. Mapping is beside the model. | +| 010 | Original and diagnostic traces retained; existing finally cleanup and failure propagation remain. | +| 011 | Shared production/model scope fault was disproved by independent adapter identities and the architecture rule; a second runtime model is unnecessary for this comparison correction. | +| 012 | Parent and source binding, evidence chronology and all applicable outcomes recorded here. | +| 013 | Sibling and same-subscription neighbors distinguish global error order from per-replay order. | +| 014 | Controlled adapter claim only; no real-provider or framework transfer claimed. | +| 015 | Both declared states exceed minimum five, with 52 and six histories respectively. | +| 016 | Matrix supplies histories to the same driver and comparison, with no separate expected error. | + +Final SHA-256 binding (the report itself is excluded): + +| Source | SHA-256 | +| --- | --- | +| `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts` | `119f75c35a0b5c472d211997a9f677fa0a92e605d99c8bde46aeda8c14a04560` | +| `packages/db/src/collection/subscription.ts` | `315a043966bf03ae7671e1e34338692d93eb91f04223a7c079a7585c4a54860b` | +| `packages/db/src/query/live/collection-config-builder.ts` | `0c00f76f05ecdea1c842e47e7749ed5dafbbb32f5c9f8a0937faf2776b4f474d` | + + +## Ordered joined dependency review (2026-10-30) + +This entry evaluates CodeRabbit review `5718630084`, run +`bbef8b96-93c0-425c-ac82-76ae3c71abec`, reviewed commit +`86510968691e0080a7484881ed7f54ba58f110af`. Its selected file was +`compiler/basic.test.ts`; the behavioral finding was outside that diff and +survived unchanged at repair parent `8464ec97725a14c9df103e170d9367737c9ef41a`. +The earlier metadata, compiler and replay evidence remains historical evidence +for its named source bindings. This entry supersedes the earlier fingerprints +of the files changed here. The final source binding below identifies this +repair without claiming that the external review covered it. + +**Law and authority.** The architecture's “Ordered requests, continuation, +and recovery” and “Atomic window publication” require ordered continuation +and publication to wait for required joined demand. Retaining two ordered +plans over one source retains both plans' joined dependencies. An outer +QueryRef that has no join cannot remove its child's dependency. A second join +cannot substitute its own source for the child's. This is enforcement of the +existing complete-window law, with no new product decision. + +**Experiment.** The primary `ordered-work-oracle.property.test.ts` owner had +compiler/cache composition coverage and joined receiving coverage separately. +It lacked their intersection: nested ordered plans while joined acquisition +is held. Its new independent judgment filters complete source rows, sorts +and slices using `expectedInstalledWindow`. It never reads the compiler plan +or production publication classifier to compute an expected result. +The supporting `ordered-joined-plan-driver.ts` uses actual live-query +Collections and Effects with controlled source adapters. Adapter request data +selects the marker rows independently through the reference expression judge. +The model's held/partial state abstracts pending acquisition and the preceding +complete public snapshot. The settled state adds the new row and recomputes +membership. Source facts for that row are deliberately unavailable until the +held adapter answers, so the promised early observation is retention, not a +speculative result that needs those missing facts. + +Before production edits, 128 finite histories crossed Collection/Effect, +ascending/descending, matched/anti-joined filters, direct/wrapped/two-filter +nested ordering, widths one/two, source answer order and the source that vetoes +new row 4. Each compares initial settled rows and every recorded change-callback +snapshot or Effect `onBatch` snapshot plus the public read at held, partial +and settled cuts. All 256 declared held/settled states reached once, minimum +one, including both receivers separately. Unchanged parent production failed +24 intended publication comparisons (12 Collection and 12 Effect), with 104 +passes and no type errors. In a wrapped anti-join, row 4 became visible while +its marker acquisition remained held, then disappeared after settlement. +Direct-query controls retained the preceding result correctly. + +The first draft also asserted atomic ordinary Effect startup callbacks. The +architecture explicitly permits progressive startup, so that assertion was +incorrect. It produced two additional failures. Before production edits, the +comparison was corrected to establish the initial settled baseline and begin +its new publication claim afterward. Both earlier logs remain; they do not +supply the authoritative RED result. The final 128-history RED is +`joined-plan-red-final.{json,log}` in the evidence directory below. + +**Repair and calibration.** The compiler stores the union of joined source +IDs when registering ordered plans. Loader continuation, Collection publication +and operation tracking, and Effect publication inspect every dependency. +The new matrix passes all 128 histories. The normal package run passes all +647 tests across the ordered owner and three affected loader/work suites, +including the original 424 owner tests and unchanged fixed/random budgets. +After final type/API-overload and local-name corrections, the same 128-history +matrix passes again. Focused no-emit TypeScript, ESLint (zero errors, five +existing warnings), Prettier, whitespace and changed-file enforcement pass. + +Restoring overwrite semantics fails 24 intended comparisons. A last-only +fallback, equivalent to retaining the new dependency when present, fails eight. +A first-only fallback fails eight. Each narrow control loses the source that +vetoes row 4 in the opposite answer-order cell. All failures are the intended +`ordered joined plan publication mismatch`, not timeouts or setup failures. +All temporary mutants were restored. This rejects both the original defect +and plausible repairs that preserve only one source. + +**Inference and limits.** The finite evidence closes the reported loss of +joined dependencies and its two-source sibling within these legal histories, +actual core receivers and publication cuts. It preserves direct controls, +settled membership, direction and width, and does not add speculative callbacks. +It covers one inserted head row, at most two joined dependencies, one QueryRef +level, widths one/two and controlled successful acquisition. It does not claim +arbitrary nesting, provider/framework timing, new rejection/cleanup laws, +public window-promise integration or atomic ordinary Effect startup. Other +ordered-work limits and existing static metadata/P11/P12 gaps remain recorded +with their prior owners. No known counterexample remains in this new matrix. + +Evidence directory: +`/private/tmp/callback-metadata/ci-monitor/20261030T210918Z`. +It contains the complete raw review/conversation bodies, append-only ledger +references, pre-edit source copies and fingerprints, diagnostic probe, +`joined-plan-red-final.*`, `joined-plan-green.*`, `joined-plan-package-green.*`, +`joined-plan-final-green.*`, three `joined-plan-mutant-*` runs and summary, +`joined-plan-types-final.log`, `joined-plan-eslint-final.log`, formatting and +changed-file enforcement logs. The matrix command is: + +```sh +pnpm_config_verify_deps_before_run=false pnpm --filter @tanstack/db exec \ + vitest run tests/query/ordered-work-oracle.property.test.ts \ + -t 'retains every joined dependency' --coverage.enabled=false --maxWorkers=1 +``` + +### Lossless review ledger and reviewer assessment + +The raw review, repeated prompt, run metadata, mutable risk prose and footnotes +were rechecked against the final diff. Six source-order items remain distinct. +The two-source sibling is calibration and repair work under item 1. + +| ID | Technical verdict and PR action | Durable value and destination | +| --- | --- | --- | +| CR5718630084-1 | **fixed-now**. Dependency loss is real. The review calls it minor; observed transient public membership makes it a correctness defect. Its nullish fallback preserves only one source and fails the wider law. | Nested held-dependency histories in the primary ordered owner, production union and this evidence record. | +| CR5718630084-2 | **duplicate** of 1. The repeated agent prompt proposes the same nullish fallback. | Preserve its exact proposal in the scratch ledger and reject it with the last-only mutant. | +| CR5718630084-3 | **accepted-design**. Optional local CLI review supplies no product requirement and needs no installation. | Existing no-install constraint and explicit oracle calibration remain the authorized verification method. | +| CR5718630084-4 | **accepted-design**. API review commit is 865; one selected test file does not establish fresh review of every production file. The outside-diff finding survives at parent 846. | Keep reviewed commit separate from repair source binding and later CI head. | +| CR5718630084-5 | **duplicate** of 1. Mutable merge-risk prose says to retain the joined dependency. | The receiving oracle now observes the transient public result described by that prose. | +| CR5718630084-6 | **duplicate** of prior review items 5–6. The unchanged 47.37% docstring threshold and optional generated tests do not identify another contract gap. | Preserve repository literate-oracle requirements and review limitations without adding unrelated generated examples. | + +Totals: one fixed-now, two accepted-design, three duplicates, zero +confirmed-open, deferred or design-decision items in this review. The final +loss audit includes prose outside the finding list, optional CLI/tests, +coverage qualification and the mutable summary. No item was removed by a +severity cap. Reviewer recommendation: **hire, with a proposed-fix qualification**. +The review found a real outside-diff composition defect with a precise cause +and useful receiving-test direction. Its priority understated public transient +behavior, and its patch missed two-source composition. The generated policy +footnotes add little new signal. Oracle experiments supply the wider closure. +This assessment does not establish whole-PR readiness or review of the repair. + +### Oracle self-review + +This is a self-review, not an independent external audit. + +| Requirement | Outcome | +| --- | --- | +| ORC-001 | Existing ordered/atomic-publication authority, bounded histories and explicit startup/provider limits above. | +| ORC-002 | Complete source truth, ordinary membership and independent sorting/slicing; no plan or gate helper computes expectations. | +| ORC-003 | Law and limits precede the finite grammar; model, named driver and shared comparison stay distinguishable beside executable prose. | +| ORC-004 | Wrapped anti-join reconstructs the finding. Direct placement removes composition and stays green. Two-filter/blocker/answer-order dimensions distinguish first-only and last-only. Directions, widths and polarities preserve rank/membership neighbors. Bounds are finite; the grammar excludes a second blocker for a plan that has only one joined source. | +| ORC-005 | Real Collection reads/change callbacks and Effect onBatch reconstruction at held, partial and settled cuts; held adapters explain missing source facts without excusing speculative publication. | +| ORC-006 | Overwrite, first-only and last-only controls fail intended public comparisons, with counts 24/eight/eight. | +| ORC-007 | New matrix is bounded enumeration, not a random campaign. Existing owner fixed/random campaigns and budgets remain and pass in the 647-test run. | +| ORC-008, ORC-009 | Pending adapter acquisition maps to held/partial cuts; retained public snapshot and settled source truth are separate model observations. No production continuation machine is copied. | +| ORC-010 | Comparison errors retain observation/history traces in AggregateError and cleanup uses withOracleCleanup. Earlier recorder-assumption failures and all mutant diagnostics remain separately available. | +| ORC-011 | No new shared semantic-fault hypothesis requires a second runtime model. Source-complete membership differs structurally from compiler dependency union and publication scheduling. | +| ORC-012 | Parent, chronology, boundaries, source bindings, loss audit and every applicable outcome are recorded here. | +| ORC-013 | No-join outer plan and two distinct joins distinguish absence preservation from union; both blocker sources and answer orders reject narrow fixes. | +| ORC-014 | Actual core receivers supply the receiving boundary; controlled adapters alone supply acquisition timing. No external provider/framework transfer is claimed. | +| ORC-015 | Every one of 128 histories reaches held and settled once: 256 cells, minimum one, per-receiver identity included in each key. | +| ORC-016 | All new cases use the same independent model, named production driver and refinement comparison; no ordinary expected-answer examples are added to the oracle. | + +Final SHA-256 binding for this repair (this report is excluded): + +| Source | SHA-256 | +| --- | --- | +| `packages/db/src/query/compiler/order-by.ts` | `265c1a8102865e8d0be70bab3b6853e757530800098c8b166545988445b4fab9` | +| `packages/db/src/query/effect.ts` | `a17ed6693a48de79a45b8602bd7803b6a577174d6d796db7038e10c964986220` | +| `packages/db/src/query/live/collection-config-builder.ts` | `fb584c9a49a68faa7775b572ec6d5e0d2ce9f968904d18000c269734c735ab76` | +| `packages/db/src/query/live/collection-subscriber.ts` | `d2e7864f197eaced0323453eda3c70c6967080b956e1ef6df266f1b73234e527` | +| `packages/db/src/query/live/ordered-source-loader.ts` | `3baf426831e184b88656c475be2e6362cd670e6f5fa6e300ea9ec2ed82bf10db` | +| `packages/db/tests/query/ordered-work-oracle.property.test.ts` | `652a1a2378ab882a3e4b0619889020f0581212653507f25051a684b60e8b1d71` | +| `packages/db/tests/query/ordered-joined-plan-driver.ts` | `3a3bb38caef14e649df17ac7cd0eaae312aa483b4801c4d874888b4112c753f9` | +| `packages/db/tests/query/ordered-source-loader-state-oracle.test.ts` | `679449468e358d51469bffe18d159ec62b8557b1c63aeaa3ea4cee2e05628b49` | +| `packages/db/tests/query/ordered-source-loader-state.test.ts` | `23e30afddbfbd5db5979a44f580ac3fc777e5c5ae216584dd7073964080c2925` | +| `packages/db/tests/query/ordered-work.test.ts` | `df0bb6faf5d81f70c1229511dd38180c206d2df0560593550a6220a8e2a2941f` | + + +## Replay retirement scope review (2026-10-30) + +CodeRabbit review `5718757637`, run `f8b20e61-196f-44d2-86fa-d7648cfa8d08`, +covered `8464ec977`. Three files were selected, two similar documentation files +were skipped, and the replay oracle was processed. The current production +repair `c33e18fd5` was not reviewed. Automatic reviews are now paused. + +**Law.** Each subscription owns its replay verdict (architecture, “Atomic +window publication” and “Replay participants and failure”). A successful +sibling acquisition cannot decide another subscription's replay failure, +even when both request the same source keys. Publication can remain shared. + +**Experiment.** The prior oracle repair scoped error comparison and covering +failures, but missed successful retirement in `settleParticipant`. The primary +owner now varies failure followed by successful coverage over 96 histories: +two directions, four join forms, three widths, root/marker replay and both +settlement orders. It uses the existing actual Collection driver and every +shared public comparison. A calibration comparison also requires every sibling +failure identity to survive that account action. Before repairing the account, +it fails with `replay account scope: successful sibling removed another +subscription's replay failure`, with no type errors. The repair adds the same +subscription premise to successful failure retirement. No production code +changes for this finding. The complete normal owner run passes all 44 tests, +including unchanged fixed/random budgets. Four histories reach successful +sibling coverage, minimum one. The prior two-failure matrix retains 52 own +and six sibling error-order histories. The original retirement behavior is +the rejected hostile control. This establishes account sensitivity, not a +new production defect or universal error-order proof. + +Evidence: `ci-monitor/20261030T212023Z/replay-retirement-{red,package-green}.*` +under `/private/tmp/callback-metadata`, original oracle copy, source ledger, +ESLint and changed-file enforcement logs. Existing controlled-adapter, peer, +provider and framework limits remain. The publication, convergence and +request-error comparisons remain, without a new waiver. + +The six-item raw-body loss audit yields one **fixed-now** (successful retirement +scope), two **accepted-design** (coverage qualification and paused review), +and three **duplicates** (repeated fix prompt, mutable risk prose, docstring +policy and optional generated docs/tests). The docstring percentage changes to +45%, but names no separate missing contract explanation. Optional review +commands supply no coverage and are not executed. No item is deferred or open. +Reviewer recommendation: **hire**. The finding correctly identifies a remaining +oracle-integrity gap and proposes the missing ownership premise. Its scope is +test correctness. It does not establish review of the later production repair. + +ORC self-review, not independent audit: 001–003 use the established ownership +law, input identities and literate shared comparison. 004 reconstructs the +failure/success neighbor of the prior failure/failure grammar, with finite +bounds and no peer claim. 005 exercises actual Collections at the existing +cuts and calibrates the oracle account separately. 006 rejects the original +retirement action at its intended assertion. 007 preserves all campaigns and +budgets. 008–009 retain per-subscription failure identities within the shared +publication abstraction. 010 retains the original trace and existing cleanup. +011 names no new shared semantic-fault hypothesis requiring a second runtime +model. 012 records chronology, source and limits here. 013 crosses failure with +success and own with sibling coverage. 014 claims controlled adapters through +real core receivers only. 015 reaches four successful sibling histories against +minimum one. 016 supplies histories to the shared driver and judgments, with +no separate expected Error examples. + +Final SHA-256 of `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts`: +`e03f3cee17413016f59c299eb25896fd9098ea3cde88fffbe35f4de58fba366f`. This supersedes that owner's earlier binding only. + + +## Shipped-size refactor (2026-10-30) + +The size request preserves the accepted metadata, cache, ordering and publication +laws. The primary owners, history grammars, comparisons and run budgets do not +change. This is a behavior-preserving refactor, with no new pre-edit RED claim. +Shared FROM/JOIN source compilation and input wrapping replace duplicate paths. +Container copying, include replacement, projection and ordering each use fewer +implementations. Authored output and normalized callback views remain distinct. +Every ordered window and joined dependency remains represented. + +`pnpm size:compare --packages db --files` measures production bundles against +merge-base `bfd1da1a7e752d66fd667adfd3cdf54f60732b22`. The repository has no +`size:check` script. Targeted builds use existing dependencies without an install. + +| Measurement | Base gzip | Refactor gzip | Gzip delta | Brotli delta | +| --- | ---: | ---: | ---: | ---: | +| Full `@tanstack/db` package | 109,702 B | 109,673 B | −29 B | −13 B | +| Typical app | 96,777 B | 96,743 B | −34 B | −26 B | + +These totals do not establish that every feature scenario shrinks. LocalStorage +adds 17 B gzip and 5 B Brotli. Some other scenarios shrink with gzip but grow +with Brotli. The complete scenario table remains in `size-union-key.log`. + +The frozen source passes 6,322 normal-package tests across 11 suites, including +both primary owners and the affected compiler, container and include neighbors. +Focused no-emit TypeScript, ESLint (zero errors and warnings), Prettier, +changed-file enforcement and whitespace checks pass. Seven source-bound hostile +controls fail their intended comparisons: lost HAVING view 551, over-normalized +predicates 696, raw-only cache 26, ordered-window overwrite 48, joined-dependency +overwrite 24, joined last-only eight and joined first-only eight. These controls +calibrate the existing laws after refactoring. They do not extend their limits. +Union key encoding also matches the original for 24 representative inputs. + +Evidence directory: +`/private/tmp/callback-metadata/ci-monitor/20261030T212503Z/size-reduction`. +The final bindings are `final-green.*`, `final-eslint2.log`, `final-types2.log`, +`final-format.log`, `final-lint-changed.log`, `size-mutant-summary.json` and +`final-fingerprints.json`. Earlier attempts remain separate. The following +bindings supersede earlier bindings for these four refactored files only. + +| Source | SHA-256 | +| --- | --- | +| `packages/db/src/query/compiler/index.ts` | `c6074cd82e6755b3d55c9307018fe368cd3e591a719eeb69ab6d7cb2109e75bf` | +| `packages/db/src/query/compiler/joins.ts` | `a5618bb1da3bff18b91fc4d9bf155447712e0bb1b68cd0f66a1cda0662d80450` | +| `packages/db/src/query/compiler/order-by.ts` | `a104d4f4829e9b4ea801fcc60914772c1d97a47d7d46dbfa5e24f49724b12267` | +| `packages/db/src/query/compiler/route-metadata.ts` | `bc6313f715e84949d0a6f8f5b48511bb3bc05ba5fd13a0b5e8a1033f3ee8e4df` | + + +### Integration with current main and final size + +The maintainer approved updating this branch from main. Merge `ce555e739` +incorporates `a2f346596c34faa7e65211134194337bd2e0007d`. The only conflict +was appended coverage-map prose. Both sides remain. This updates the branch +without merging PR #2312. + +The final compiler removes two redundant branches. `Object.is` already accepts +two `NaN` values, so its later `Number.isNaN` case cannot change the answer. +Object spread already accepts `undefined`, so parent-context copying needs no +separate empty-object case. Accepted laws, oracle grammars, comparisons and +budgets remain unchanged. + +The default `pnpm size:compare --packages db --files` now compares the integrated +tree with current main `a2f346596`, rather than the earlier merge-base. + +| Measurement | Main gzip | Final gzip | Gzip delta | Brotli delta | +| --- | ---: | ---: | ---: | ---: | +| Full `@tanstack/db` package | 109,658 B | 109,601 B | −57 B | −48 B | +| Typical app | 96,591 B | 96,529 B | −62 B | −31 B | + +Every measured scenario shrinks with gzip. The Effect scenario grows by one +byte with Brotli. The earlier original-base and exact-base diagnostics remain +historical measurements of their named trees. + +The frozen integrated source passes 6,533 tests across 16 files, including the +primary owners, restart publication, replay, distinct collisions and include +receivers. Focused no-emit TypeScript and ESLint pass, with zero lint warnings. +Evidence is `/private/tmp/callback-metadata/ci-monitor/size-final-integration`: +`size.log`, `green.*`, `types.log`, `eslint.log` and `final-fingerprints.json`. +The refreshed compiler index binding below supersedes its previous binding. +Other compiler sources retain the previous refactor bindings. + +`packages/db/src/query/compiler/index.ts`: `c356929cce3db647ec28f79ff2dd14bead69450ac6567176ec0bb63d6b8d970d`. + + +All seven controls reject the intended wrong behavior on the frozen integrated +source: HAVING view loss 551, over-normalized predicates 696, raw-only cache 26, +window overwrite 48, joined overwrite 24, joined last-only eight and joined +first-only eight. `size-mutant-summary.json` retains the assertion errors and +source hashes. The missing-dependency setup attempt remains separately in +`setup-failure-1`; it supplies no RED evidence. Existing dependency links +resolved that copied-tree setup. No install or production mutation occurred. +The isolated copies were restored and both trees match the frozen fingerprints. + +## Release-blocking loading sibling replay port (2026-10-30) + +The release workflow for main `e75f8455a78bc33c43376288218e9912dc2a3f4c` exposed a retention counterexample that also reproduces on callback head `5af7624937288cbe212f19ff23e8ba83caabb74e`. Independent adapter receipts show that a new include demand failed while a sibling replay was still loading. The include had no acquisition at truncate, and its failed demand remained held. This is an enforcement gap in architecture law 9, not evidence of a regression in the compiler removals. + +[PR 2318](https://github.com/TanStack/db/pull/2318) carries the main-based repair and its [versioned experiment record](2026-10-30-release-replay-retention.md). The same primary extension ran on unchanged callback production first: `callback-primary-red.json` fails at quiet retention with measured sibling failure 1. The one-condition guard then changed from failed replacement to pending replacement. `callback-primary-green.json` passes all 48 finite histories with reach 44/33/12. The full primary owner passes all 45 tests on this branch. The old guard's pre-edit RED and the main-based original-guard mutant preserve the intended violated comparison. + +The SQLite grammar port is byte-identical to the main-based fix. Its two 80-history campaigns and two existing final-delivery controls pass there. This branch's focused TypeScript, ESLint, formatting, whitespace and `pnpm lint:changed` pass. ESLint retains two existing warnings. All 14 compiler source fingerprints from the final size integration remain unchanged. The bounded core/controlled-host claims and historical recovery-reach gate limitation remain as stated in the experiment record. + +After the guard port, `pnpm_config_verify_deps_before_run=false pnpm size:compare --packages db --files` reports package gzip 109658 to 109595 bytes (-63), Brotli -47, against merge-base `a2f346596`. Typical app gzip changes from 96591 to 96527 (-64), with Brotli +21. Every measured gzip scenario shrinks. This does not claim every Brotli scenario shrinks. The later release merge changes versions and changelogs rather than these runtime sources. + +Scratch evidence is under `/private/tmp/callback-metadata/release-replay-repair`. These SHA256 fingerprints bind the callback port separately from the main-based repair: + +| File | SHA256 | +| --- | --- | +| `packages/db/src/query/live/collection-config-builder.ts` | `5ef5db6bde45a6a34aa54706772041653996bd1c3e974d0dcb7104884f12cfb0` | +| `packages/db/src/collection/subscription.ts` | `bd171bdc97b38a79ebbc81d7fb605267787308559ad4c0646dff1e072a43798a` | +| `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts` | `e5ce4a77fe8d16f4b0d26f4c71fdadb53b2779c8e7e8313a7eb32d2de3ebdd38` | +| `packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts` | `67c16d284522417f1a77e0a36efcc552b718f858ea8ec00771f001f001160782` | + + +### Subscription-scoped request-error checker port (2026-10-30) + +Review 5719714297 on repair PR 2318 identified a primary-oracle exemption that +combined sibling subscriptions by source keys. The same unchanged classifier +was present here. The oracle-only repair retains subscription identity for +later requests and coverage, and measures the owed public health comparison in +the existing 96-history grammar. The shared +[versioned amendment](2026-10-30-release-replay-retention.md#subscription-scoped-request-error-checker-repair) +records the accepted law, old/new distinguishing probe, targeted omitted-error +calibration, intermediate zero-reach result and limits. This port changes no +runtime source, browser grammar, shipped-size result or compiler fingerprint. diff --git a/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts index f109c45c08..341715569a 100644 --- a/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts @@ -989,24 +989,53 @@ describe(`multi-tab Electric over one persisted database`, () => { }), steps: fc.array(historyStep, { minLength: 8, maxLength: 24 }), }) - .map(({ tabCount, source, key, recovery, stopOrder, steps }) => { - const tab = source % tabCount - const peer = (tab + 1) % tabCount - return { + .map( + ({ tabCount, - steps: [ - ...stopOrder.flatMap((pick): Array => [ + source, + key, + recovery, + stopOrder, + steps, + }): GeneratedHistory => { + const tab = source % tabCount + const peer = (tab + 1) % tabCount + return { + tabCount, + steps: [ + ...stopOrder.flatMap((pick): Array => [ + { kind: `server change`, key, pick: 0 }, + { kind: `advance`, tab, pick: 0 }, + { kind: `stop`, tab, pick }, + { kind: `catch up`, tab: peer, pick: 0 }, + { kind: `recover`, tab, pick: recovery }, + { kind: `catch up`, tab, pick: 0 }, + ]), + ...steps, + // Final catch-up must encounter an abandoned reservation in every + // generated history. Start a fresh handle, recover every tab, and + // deliver one new change without committing it. Reset then abandons + // that reservation. No later history step commits or recovers it; + // the existing final-delivery comparison must observe its failure + // before the existing recovery establishes convergence (Laws 2/5). + // The source tab, key and two/three-tab host vary with this grammar. + // Removing advance or reset loses that premise; catching up here + // would move the failure out of the final observation cut. + { kind: `deploy` }, + ...Array.from( + { length: tabCount }, + (_, index): Array => [ + { kind: `recover`, tab: index, pick: recovery }, + { kind: `catch up`, tab: index, pick: 0 }, + ], + ).flat(), { kind: `server change`, key, pick: 0 }, { kind: `advance`, tab, pick: 0 }, - { kind: `stop`, tab, pick }, - { kind: `catch up`, tab: peer, pick: 0 }, - { kind: `recover`, tab, pick: recovery }, - { kind: `catch up`, tab, pick: 0 }, - ]), - ...steps, - ], - } - }) + { kind: `peer reset` }, + ], + } + }, + ) type GeneratedHistory = { tabCount: number; steps: Array } type ServerLog = { diff --git a/packages/db/package.json b/packages/db/package.json index 1b8a299d76..02ebf41d16 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -23,7 +23,7 @@ "test": "vitest --run", "test:dist": "vitest run --config vitest.dist.config.ts", "test:facade-retention": "node --expose-gc --import tsx tests/facade-retention.probe.ts", - "test:oracles": "vitest --run --coverage.enabled=false tests/reverse-index-oracle.property.test.ts tests/query/compiler/in-evaluator-oracle.property.test.ts tests/cleanup-queue-oracle.property.test.ts tests/comparison-oracle.property.test.ts tests/duration-query-equality-oracle.test.ts tests/cursor-oracle.property.test.ts tests/index-update-oracle.property.test.ts tests/utils-oracle.property.test.ts tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/btree-map-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/local-storage-order-oracle.test.ts tests/local-storage-value-fidelity-oracle.property.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/collection-gc-transaction-oracle.property.test.ts tests/collection-gc-retention-oracle.property.test.ts tests/collection-gc-mutation-oracle.property.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/transaction-ownership-oracle.property.test.ts tests/optimistic-settlement-boundaries-oracle.test.ts tests/optimistic-history-publication-oracle.test.ts tests/optimistic-history-outcomes-oracle.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-index-value-oracle.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-subscription-lifecycle-history-oracle.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication-oracle.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-cache-options-oracle.test.ts tests/mutation-confirmation-oracle.test.ts tests/mutation-handler-types-oracle.test.ts tests/live-query-observer-history-oracle.property.test.ts tests/live-query-observer-hydration-oracle.property.test.ts tests/live-query-observer-snapshot-oracle.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/custom-index-order-selection-oracle.test.ts tests/query/mixed-index-order-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-alias-shadowing-oracle.test.ts tests/query/includes-outer-join-route-work-oracle.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state-oracle.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/virtual-row-fields-effect-oracle.test.ts tests/query/virtual-row-fields-types-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts tests/query/pooled-live-query-oracle.property.test.ts tests/live-query-deferred-acquisition-oracle.test.ts tests/flat-change-tracking-oracle.property.test.ts tests/truncate-readiness-oracle.test.ts tests/proxy-detachment-contract-oracle.test.ts tests/production-error-messages-oracle.test.ts tests/query/bucket-facade-rollback-oracle.property.test.ts tests/query/aggregate-select-oracle.property.test.ts tests/query/load-subset-joint-replay-oracle.property.test.ts", + "test:oracles": "vitest --run --coverage.enabled=false tests/reverse-index-oracle.property.test.ts tests/query/compiler/in-evaluator-oracle.property.test.ts tests/cleanup-queue-oracle.property.test.ts tests/comparison-oracle.property.test.ts tests/duration-query-equality-oracle.test.ts tests/cursor-oracle.property.test.ts tests/index-update-oracle.property.test.ts tests/utils-oracle.property.test.ts tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/btree-map-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/local-storage-order-oracle.test.ts tests/local-storage-value-fidelity-oracle.property.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/collection-gc-transaction-oracle.property.test.ts tests/collection-gc-retention-oracle.property.test.ts tests/collection-gc-mutation-oracle.property.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/transaction-ownership-oracle.property.test.ts tests/optimistic-settlement-boundaries-oracle.test.ts tests/optimistic-history-publication-oracle.test.ts tests/optimistic-history-outcomes-oracle.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-index-value-oracle.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-subscription-lifecycle-history-oracle.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication-oracle.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-cache-options-oracle.test.ts tests/mutation-confirmation-oracle.test.ts tests/mutation-handler-types-oracle.test.ts tests/live-query-observer-history-oracle.property.test.ts tests/live-query-observer-hydration-oracle.property.test.ts tests/live-query-observer-snapshot-oracle.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/custom-index-order-selection-oracle.test.ts tests/query/mixed-index-order-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-alias-shadowing-oracle.test.ts tests/query/includes-outer-join-route-work-oracle.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state-oracle.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/virtual-row-fields-input-oracle.test.ts tests/query/virtual-row-fields-effect-oracle.test.ts tests/query/virtual-row-fields-types-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts tests/query/pooled-live-query-oracle.property.test.ts tests/live-query-deferred-acquisition-oracle.test.ts tests/flat-change-tracking-oracle.property.test.ts tests/truncate-readiness-oracle.test.ts tests/proxy-detachment-contract-oracle.test.ts tests/production-error-messages-oracle.test.ts tests/query/bucket-facade-rollback-oracle.property.test.ts tests/query/aggregate-select-oracle.property.test.ts tests/query/load-subset-joint-replay-oracle.property.test.ts", "bench:nested-includes": "vitest bench tests/query/includes-performance.bench.ts --run", "test:indexed-db:package": "pnpm --filter @tanstack/db... build && vitest run --config vitest.indexed-db.package.config.ts", "test:collection-gc:browser": "playwright test --config playwright.collection-gc.config.ts", diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 428dd2fac8..949e0eb4d4 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -871,7 +871,7 @@ export class CollectionSubscription /** * A lazy demand that failed outside this subscription's replay, while a - * failed replay of another of the graph's subscriptions to this collection + * replay of another of the graph's subscriptions to this collection still * holds publication, fails this subscription's replacement too. It keeps * the last publication until every demand load that joins this replay * loads, a later replay succeeds, or this subscription's last demand diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 2fe86d06ad..150a8c0e20 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -19,7 +19,6 @@ import { getParentContextValue, } from '../equality-value-identity.js' import { - CollectionInputNotFoundError, DistinctRequiresSelectError, FnSelectWithGroupByError, GroupingOperatorRequiredError, @@ -56,7 +55,6 @@ import { getWhereExpression, isBasicOrAggregateExpression, isExpressionLike, - requireCollectionSource, } from '../ir.js' import { ensureIndexForField } from '../../indexes/auto-index.js' import { createSourceRecord } from '../../utils/source-record.js' @@ -68,23 +66,27 @@ import { isCaseWhenConditionTrue, toBooleanPredicate, } from './evaluators.js' -import { processJoins, registerLazyDemandPlan } from './joins.js' +import { processJoins, processSource, registerLazyDemandPlan } from './joins.js' import { findAggregate } from './find-aggregate.js' import { getLazyLoadTargets } from './lazy-targets.js' import { processOrderBy } from './order-by.js' import { crossJoinParentRoutes } from './parent-routes.js' -import { queriesMatchForCaching } from './query-equivalence.js' import { + CALLBACK_INPUT, + CALLBACK_SELECTED, INCLUDES_PUBLIC_KEY, INCLUDES_ROUTING, attachRouteMetadata, attachRouteMetadataToResult, + getCallbackInput, getNamespacedRouteMetadata, getRouteMetadata, getRoutedScalarMetadata, + isPublicContainer, stripInternalCallbackMetadata, stripInternalRouteMetadata, - stripRouteMetadata, + transformPublicContainers, + wrapInputRow, } from './route-metadata.js' import { processSelect } from './select.js' import type { ValueIdentity } from '../equality-value-identity.js' @@ -140,12 +142,9 @@ function getUnsupportedFnSelectResultDescription( } seen.add(value) - const keys = [ - ...Object.keys(value), - ...Object.getOwnPropertySymbols(value).filter((key) => - Object.prototype.propertyIsEnumerable.call(value, key), - ), - ] + const keys = Reflect.ownKeys(value).filter((key) => + Object.prototype.propertyIsEnumerable.call(value, key), + ) for (const key of keys) { const entry = (value as Record)[key] const unsupported = getUnsupportedFnSelectResultDescription(entry, seen) @@ -190,9 +189,9 @@ function projectParentContext( valueIdentity: ValueIdentity, ): Record { const inherited = getRouteMetadata(nsRow)?.parentContext - const inheritedValue = getParentContextValue(inherited) - const parentContext: Record = - inheritedValue === undefined ? {} : { ...inheritedValue } + const parentContext: Record = { + ...getParentContextValue(inherited), + } const projectedBindings = { ...getParentContextProjectedBindings(inherited), } @@ -278,25 +277,11 @@ function parameterizeByParentRoutes( ) as NamespacedAndKeyedStream } -function getRowCorrelationKey(row: NamespacedRow, mainSource: string): unknown { - return getNamespacedRouteMetadata(row, mainSource)?.correlationKey -} - -function getRowParentContext(row: NamespacedRow, mainSource: string): unknown { - return getNamespacedRouteMetadata(row, mainSource)?.parentContext ?? null -} - function correlationValuesEqual(left: unknown, right: unknown): boolean { if (left == null || right == null) return false const normalizedLeft = normalizeValue(left) const normalizedRight = normalizeValue(right) - return ( - Object.is(normalizedLeft, normalizedRight) || - (typeof normalizedLeft === `number` && - typeof normalizedRight === `number` && - Number.isNaN(normalizedLeft) && - Number.isNaN(normalizedRight)) - ) + return Object.is(normalizedLeft, normalizedRight) } /** @@ -372,6 +357,8 @@ export interface CompilationResult { includes?: Array } +const NO_OMITTED_KEYS: ReadonlySet = new Set() + const valueIdentitiesByCache = new WeakMap() function getCompilationValueIdentity(cache: QueryCache): ValueIdentity { @@ -435,11 +422,14 @@ function compileQueryInternal( ancestorHasTopK = false, ]: CompileQueryArgs, scopeValidated: boolean, - publishCollectionRoot = false, + sanitizeSelectedMetadata = false, ): CompilationResult { - // Check if the original raw query has already been compiled + // Authored and source views have different selected-row metadata. Reuse a raw + // query's compilation only within the same mode. const cachedResult = - parentKeyStream === undefined ? cache.get(rawQuery) : undefined + parentKeyStream === undefined + ? cache.get(rawQuery)?.[sanitizeSelectedMetadata ? 1 : 0] + : undefined if (cachedResult) { return cachedResult } @@ -460,6 +450,8 @@ function compileQueryInternal( ) // Use a mutable binding so we can shallow-clone select before includes mutation let query = optimizedQuery + const sanitizeSourceMetadata = + sanitizeSelectedMetadata || query.fnSelect !== undefined // Create mapping from optimized query to original for caching queryMapping.set(query, rawQuery) @@ -509,7 +501,9 @@ function compileQueryInternal( aliasRemapping, sourceWhereClauses, parentKeyStream, - publishCollectionRoot, + // Sanitize source rows before a callback consumes them. Inline values + // without a callback keep their authored selected shape. + sanitizeSourceMetadata, ) Object.assign(sources, fromSources) const sourceCarriesInternalRouteState = @@ -536,12 +530,12 @@ function compileQueryInternal( // child value beside the row so result routing can still expose it. const childFieldPath = childCorrelationField.path.slice(1) // remove alias prefix const childRekeyed = mainInput.pipe( - map(([key, row]: [unknown, any]) => { + map(([key, row, callbackValue]: [...InputRow, unknown?]) => { const correlationValue = getNestedValue(row, childFieldPath) return [ valueIdentity.serializeEquality(correlationValue), - [key, row, correlationValue], - ] as [unknown, [unknown, any, unknown]] + [key, row, correlationValue, callbackValue], + ] as [unknown, [unknown, any, unknown, unknown?]] }), ) @@ -568,7 +562,7 @@ function compileQueryInternal( return childSide != null }), map(([_correlationIdentity, [childSide, parentSide]]: any) => { - const [childKey, childRow, correlationValue] = childSide + const [childKey, childRow, correlationValue, callbackValue] = childSide const tagged: any = attachRouteMetadata( { ...childRow, @@ -584,7 +578,7 @@ function compileQueryInternal( getParentContextIdentity(parentSide), ]) : childKey - return [effectiveKey, tagged] + return [effectiveKey, tagged, callbackValue] as unknown as InputRow }), ) @@ -612,17 +606,30 @@ function compileQueryInternal( mainCollectionId, mainSource, allInputs, - cache, queryMapping, collections, - subscriptions, callbacks, lazySources, - optimizableOrderByCollections, - ignoreWindow, rawQuery, query.from, - (...args) => compileQueryInternal(args, true, publishCollectionRoot), + (subquery, parentRoutes) => + compileQueryInternal( + [ + subquery, + allInputs, + collections, + subscriptions, + callbacks, + lazySources, + optimizableOrderByCollections, + ignoreWindow, + cache, + queryMapping, + parentRoutes, + ], + true, + sanitizeSourceMetadata, + ), aliasToCollectionId, aliasRemapping, sourceWhereClauses, @@ -648,7 +655,7 @@ function compileQueryInternal( filter(([, row]) => correlationValuesEqual( compiledChildCorrelation(row), - getRowCorrelationKey(row, mainSource), + getNamespacedRouteMetadata(row, mainSource)?.correlationKey, ), ), ) as NamespacedAndKeyedStream @@ -673,9 +680,9 @@ function compileQueryInternal( for (const fnWhere of query.fnWhere) { pipeline = pipeline.pipe( filter(([_key, namespacedRow]) => { - const callbackRow = sourceCarriesInternalRouteState - ? (stripInternalCallbackMetadata(namespacedRow) as NamespacedRow) - : namespacedRow + const callbackRow = stripInternalCallbackMetadata( + getCallbackInput(namespacedRow), + ) as NamespacedRow return toBooleanPredicate(fnWhere(callbackRow)) }), ) @@ -959,7 +966,7 @@ function compileQueryInternal( // Replace includes entry in select with a null placeholder query = { ...query, - select: replaceIncludesInSelectValue(query.select!, path).value, + select: replaceIncludesInSelectValue(query.select!, path), } } } @@ -1043,46 +1050,40 @@ function compileQueryInternal( // Process the SELECT clause early - always create $selected // This eliminates duplication and allows for DISTINCT implementation if (query.fnSelect) { - const fnSelect = (row: NamespacedRow) => { - const selected = query.fnSelect!(row) - validateFnSelectResult(selected) - return selected - } const sourceAliases = [...sourceBindings.keys()] // Handle functional select - apply the function to transform the row - const projectRow = (namespacedRow: NamespacedRow) => { - const callbackRow = sourceCarriesInternalRouteState - ? (stripInternalCallbackMetadata(namespacedRow) as NamespacedRow) - : namespacedRow - const selectResults = fnSelect(callbackRow) - let selected = selectResults - if ( - selectResults && - typeof selectResults === `object` && - (Array.isArray(selectResults) || isPlainObject(selectResults)) - ) { - selected = Array.isArray(selectResults) - ? [...selectResults] - : { ...selectResults } - if ( - publishCollectionRoot && - !sourceAliases.some((alias) => selectResults === callbackRow[alias]) - ) { - // New functional objects are projections. Equal field values cannot - // prove which source (if any) authored them. - for (const prop of VIRTUAL_PROP_NAMES) delete selected[prop] - } - const routing = (namespacedRow as any)[INCLUDES_ROUTING] - if (routing) { - selected[INCLUDES_ROUTING] = routing + pipeline = pipeline.pipe( + map(([key, namespacedRow]) => { + const callbackRow = stripInternalCallbackMetadata( + getCallbackInput(namespacedRow), + ) as NamespacedRow + const selectResults = query.fnSelect!(callbackRow) + validateFnSelectResult(selectResults) + let selected = selectResults + if (isPublicContainer(selectResults)) { + selected = copySelectedRow(selectResults) + const routing = (namespacedRow as any)[INCLUDES_ROUTING] + if (routing) selected[INCLUDES_ROUTING] = routing } - } - return { - ...namespacedRow, - $selected: selected, - } - } - pipeline = pipeline.pipe(map(([key, row]) => [key, projectRow(row)])) + // A returned source row keeps its genuine metadata. A new functional + // projection has no source owner; use the same copy-on-write metadata + // normalization as structured selections for its callback view. + const callbackSelected = sourceAliases.some( + (alias) => selectResults === callbackRow[alias], + ) + ? selected + : attachVirtualPropsToSelected(selected, {}, true, []) + if (sanitizeSelectedMetadata) selected = callbackSelected + return [ + key, + { + ...namespacedRow, + $selected: selected, + [CALLBACK_SELECTED]: callbackSelected, + }, + ] + }), + ) } else if (query.select) { // An aggregate query computes its select values from each group, so it // does not project rows first. @@ -1116,9 +1117,7 @@ function compileQueryInternal( if (includesRoutingFns.length > 0) { pipeline = pipeline.pipe( map(([key, namespacedRow]: any) => { - const selected = Array.isArray(namespacedRow.$selected) - ? [...namespacedRow.$selected] - : { ...namespacedRow.$selected } + const selected = copySelectedRow(namespacedRow.$selected) selected[INCLUDES_ROUTING] = getRowIncludesRouting(namespacedRow) return [key, { ...namespacedRow, $selected: selected }] }), @@ -1189,10 +1188,9 @@ function compileQueryInternal( for (const fnHaving of query.fnHaving) { pipeline = pipeline.pipe( filter(([_key, namespacedRow]) => { - const callbackRow = - sourceCarriesInternalRouteState || includesRoutingFns.length > 0 - ? (stripInternalCallbackMetadata(namespacedRow) as NamespacedRow) - : namespacedRow + const callbackRow = stripInternalCallbackMetadata( + getCallbackInput(namespacedRow), + ) as NamespacedRow return fnHaving(callbackRow) }), ) @@ -1227,35 +1225,57 @@ function compileQueryInternal( : [mainSource] const finalizeRow = ( key: unknown, - row: Record, + row: Record, orderByIndex: string | undefined, ) => { - const finalResults = - query.fnSelect && publishCollectionRoot - ? unwrapValue(row.$selected) + const selectedValue = unwrapValue(row.$selected) + const authoredResults = + query.fnSelect && sanitizeSelectedMetadata + ? selectedValue : attachVirtualPropsToSelected( - unwrapValue(row.$selected), + selectedValue, row, - publishCollectionRoot, + sanitizeSelectedMetadata, metadataSource, ) - // When in includes mode, embed the correlation key and parentContext - if (parentKeyStream) { - return [ - key, - [ - stripInternalRouteMetadata(finalResults), - orderByIndex, - getRowCorrelationKey(row, mainSource), - getRowParentContext(row, mainSource), - getIncludesPublicKey(row, mainSource, key), - ], - ] as any - } - return [key, [finalResults, orderByIndex]] as [ - unknown, - [any, string | undefined], - ] + const finalResults = stripInternalRouteMetadata(authoredResults) + const callbackRow = getCallbackInput(row) + const sourceViews = Object.entries(callbackRow).filter( + ([alias, value]) => alias !== `$selected` && value !== row[alias], + ) + const callbackSelected = query.fnSelect + ? (row as { [CALLBACK_SELECTED]?: unknown })[CALLBACK_SELECTED] + : attachVirtualPropsToSelected( + transformPublicContainers( + selectedValue, + (value) => + sourceViews.find(([alias]) => value === row[alias])?.[1] ?? value, + NO_OMITTED_KEYS, + ), + callbackRow, + true, + metadataSource, + ) + const route = parentKeyStream + ? getNamespacedRouteMetadata(row, mainSource) + : undefined + // Carry both views with the occurrence. A downstream source adapter selects + // the callback view without rewriting the authored inline result. + return [ + key, + [ + finalResults, + orderByIndex, + route?.correlationKey, + parentKeyStream ? (route?.parentContext ?? null) : undefined, + parentKeyStream + ? (row[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? + row[INCLUDES_PUBLIC_KEY] ?? + key) + : undefined, + stripInternalRouteMetadata(callbackSelected), + ], + ] as unknown as [unknown, [any, string | undefined]] } // A single-result query must expose only its first row when another query @@ -1278,14 +1298,9 @@ function compileQueryInternal( parentKeyStream && (resultLimit !== undefined || query.offset !== undefined) ? (_key: unknown, row: unknown) => { - const correlationKey = getRowCorrelationKey( - row as NamespacedRow, - mainSource, - ) - const parentContext = getRowParentContext( - row as NamespacedRow, - mainSource, - ) + const route = getNamespacedRouteMetadata(row, mainSource) + const correlationKey = route?.correlationKey + const parentContext = route?.parentContext if (parentContext != null) { return serializeValue([ valueIdentity.equality(correlationKey), @@ -1322,7 +1337,7 @@ function compileQueryInternal( ) as ResultStream } - // Cache the result before returning (use original query as key) + // Cache the result by the original query and selected-row metadata mode. const compilationResult: CompilationResult = { collectionId: mainCollectionId, pipeline: resultPipeline, @@ -1332,7 +1347,13 @@ function compileQueryInternal( aliasRemapping, includes: includesResults.length > 0 ? includesResults : undefined, } - if (parentKeyStream === undefined) cache.set(rawQuery, compilationResult) + if (parentKeyStream === undefined) { + getOrCreate( + cache, + rawQuery, + (): [CompilationResult?, CompilationResult?] => [], + )[sanitizeSelectedMetadata ? 1 : 0] = compilationResult + } return compilationResult } @@ -1348,10 +1369,9 @@ function keyWhereClausesBySource( query: QueryIR, clauses: Map>, ): Map> { - const localSources = [ - ...getFromSources(query.from), - ...(query.join?.map(({ from }) => from) ?? []), - ].filter((source): source is CollectionRef => source.type === `collectionRef`) + const localSources = getAllSources(query).filter( + (source): source is CollectionRef => source.type === `collectionRef`, + ) const localSourceByAlias = new Map( localSources.map((source) => [source.alias, source]), ) @@ -1385,17 +1405,20 @@ function canonicalizeSelectedRows( const compiledOrder = (query.orderBy ?? []).map(({ expression }) => compileExpression(expression), ) - const signature = (row: any) => ({ - value: row.$selected, - routing: row.$selected?.[INCLUDES_ROUTING], - outerCorrelation: isIncludedRelation - ? getRowCorrelationKey(row, mainSource) - : undefined, - parentContext: isIncludedRelation - ? getRowParentContext(row, mainSource) - : undefined, - order: compiledOrder.map((evaluate) => evaluate(row)), - }) + const signature = (row: any) => { + const route = isIncludedRelation + ? getNamespacedRouteMetadata(row, mainSource) + : undefined + return { + value: row.$selected, + routing: row.$selected?.[INCLUDES_ROUTING], + outerCorrelation: route?.correlationKey, + parentContext: isIncludedRelation + ? (route?.parentContext ?? null) + : undefined, + order: compiledOrder.map((evaluate) => evaluate(row)), + } + } return pipeline.pipe( reduce((values: Array<[any, number]>) => { @@ -1597,7 +1620,7 @@ function processFromClause( aliasRemapping: Record, sourceWhereClauses: Map>, parentKeyStream?: KeyedStream, - publishCollectionRoot = false, + sanitizeSelectedMetadata = false, ): { alias: string pipeline: NamespacedAndKeyedStream @@ -1625,29 +1648,55 @@ function processFromClause( aliasRemapping, sourceWhereClauses, parentKeyStream, - publishCollectionRoot, + sanitizeSelectedMetadata, + ) + } + + const compileSource = (source: CollectionSourceRef | QueryRef) => { + const result = processSource( + source, + allInputs, + queryMapping, + (subquery, parentRoutes) => + compileQueryInternal( + [ + subquery, + allInputs, + collections, + subscriptions, + callbacks, + lazySources, + optimizableOrderByCollections, + setWindowFn, + cache, + queryMapping, + parentRoutes, + ], + true, + sanitizeSelectedMetadata, + ), + aliasToCollectionId, + aliasRemapping, + sourceWhereClauses, + valueIdentity, + `from`, + parentKeyStream, ) + return { + ...result, + sourceIncludes: + result.includes?.map((include) => ({ + sourceAlias: source.alias, + include, + })) ?? [], + isParentRouted: + source.type === `queryRef` && parentKeyStream !== undefined, + } } if (from.type !== `unionFrom`) { const { alias, input, collectionId, sourceIncludes, isParentRouted } = - processFrom( - from, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - aliasToCollectionId, - aliasRemapping, - sourceWhereClauses, - parentKeyStream, - publishCollectionRoot, - ) + compileSource(from) return { alias, @@ -1678,23 +1727,7 @@ function processFromClause( collectionId, sourceIncludes: childSourceIncludes, isParentRouted, - } = processFrom( - source, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - aliasToCollectionId, - aliasRemapping, - sourceWhereClauses, - parentKeyStream, - publishCollectionRoot, - ) + } = compileSource(source) if (!mainAlias) { mainAlias = alias @@ -1762,7 +1795,7 @@ function processUnionAll( aliasRemapping: Record, sourceWhereClauses: Map>, parentKeyStream?: KeyedStream, - publishCollectionRoot = false, + sanitizeSelectedMetadata = false, ): { alias: string pipeline: NamespacedAndKeyedStream @@ -1813,7 +1846,7 @@ function processUnionAll( undefined, ], true, - publishCollectionRoot, + sanitizeSelectedMetadata, ) if (!mainCollectionId) { @@ -1829,8 +1862,17 @@ function processUnionAll( const branchPipeline = branchResult.pipeline.pipe( map((data: any) => { - const [key, [row, _order, correlationKey, parentContext, publicKey]] = - data + const [ + key, + [ + row, + _order, + correlationKey, + parentContext, + publicKey, + callbackValue, + ], + ] = data const branchKey = `${index}:${encodeKeyForUnionBranch(key)}` const branchPublicKey = `${index}:${encodeKeyForUnionBranch( publicKey ?? key, @@ -1843,7 +1885,13 @@ function processUnionAll( branchPublicKey, ) : row - return [branchKey, routedRow] as [string, Record] + const namespaced = + callbackValue !== undefined && + callbackValue !== row && + isPlainObject(routedRow) + ? { ...routedRow, [CALLBACK_INPUT]: [callbackValue] } + : routedRow + return [branchKey, namespaced] as [string, Record] }), ) @@ -1869,215 +1917,20 @@ function wrapInputWithAlias( alias: string, ): NamespacedAndKeyedStream { return input.pipe( - map(([key, row]) => { - const inputRow: unknown = row - const scalar = getRoutedScalarMetadata(inputRow) - if (scalar) { - const nsRow = attachRouteMetadata( - { - ...getParentContextValue(scalar.parentContext), - [alias]: scalar.value, - [INCLUDES_PUBLIC_KEY]: scalar.publicKey, - }, - scalar.correlationKey, - scalar.parentContext, - ) as unknown as NamespacedRow - return [key, nsRow] as [unknown, NamespacedRow] - } - - if (inputRow == null || typeof inputRow !== `object`) { - return [key, { [alias]: inputRow }] as [unknown, NamespacedRow] - } - - // Initialize the record with a nested structure. Route metadata remains - // outside the user namespace while projected parent aliases stay visible. - const route = getRouteMetadata(inputRow) - const cleanRow = route - ? stripRouteMetadata(inputRow as Record) - : inputRow - const nsRow: Record = { - ...getParentContextValue(route?.parentContext), - [alias]: cleanRow, - } - if (route) { - attachRouteMetadata(nsRow, route.correlationKey, route.parentContext) - } - return [key, nsRow] as [unknown, Record] - }), + map(([key, row, callbackValue]: [...InputRow, unknown?]) => [ + key, + wrapInputRow(alias, row, callbackValue), + ]), ) } function encodeKeyForUnionBranch(key: unknown): string { - if (typeof key === `string`) { - return `string:${key}` - } - if (typeof key === `number`) { - return `number:${String(key)}` - } - if (typeof key === `bigint`) { - return `bigint:${String(key)}` - } - return `${typeof key}:${JSON.stringify(key)}` -} - -function processFrom( - from: CollectionSourceRef | QueryRef, - allInputs: Record, - collections: Record, - subscriptions: Record, - callbacks: Record, - lazySources: Set, - optimizableOrderByCollections: Record, - setWindowFn: (windowFn: (options: WindowOptions) => void) => void, - cache: QueryCache, - queryMapping: QueryMapping, - aliasToCollectionId: Record, - aliasRemapping: Record, - sourceWhereClauses: Map>, - parentKeyStream?: KeyedStream, - publishCollectionRoot = false, -): { - alias: string - input: KeyedStream - collectionId: string - sourceIncludes: Array - isParentRouted: boolean -} { - switch (from.type) { - case `descriptorRef`: - return requireCollectionSource(from) - case `collectionRef`: { - const input = allInputs[from.sourceId] - if (!input) { - throw new CollectionInputNotFoundError( - from.alias, - from.collection.id, - Object.keys(allInputs), - ) - } - aliasToCollectionId[from.alias] = from.collection.id - return { - alias: from.alias, - input, - collectionId: from.collection.id, - sourceIncludes: [], - isParentRouted: false, - } - } - case `queryRef`: { - // Preserve the user-defined query as the cache key when optimization - // only copied it. If the optimizer changed the query, compile that IR; - // substituting its origin would discard pushed predicates. - const originalQuery = queryMapping.get(from.query) - const queryToCompile = - originalQuery && queriesMatchForCaching(from.query, originalQuery) - ? originalQuery - : from.query - const subQueryResult = compileQueryInternal( - [ - queryToCompile, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - parentKeyStream, - undefined, - ], - true, - publishCollectionRoot, - ) - - // Pull up alias mappings from subquery to parent scope. - // This includes both the innermost alias-to-collection mappings AND - // any existing remappings from nested subquery levels. - Object.assign(aliasToCollectionId, subQueryResult.aliasToCollectionId) - Object.assign(aliasRemapping, subQueryResult.aliasRemapping) - - // Pull up source WHERE clauses from subquery to parent scope. - // This enables loadSubset to receive the correct where clauses for subquery collections. - // - // IMPORTANT: Skip pull-up for optimizer-created subqueries. These are detected when: - // 1. The outer alias (from.alias) matches the inner alias (from.query.from.alias) - // 2. The subquery was found in queryMapping (it's a user-defined subquery, not optimizer-created) - // - // For optimizer-created subqueries, the parent already has the sourceWhereClauses - // extracted from the original raw query, so pulling up would be redundant. - // More importantly, pulling up for optimizer-created subqueries can cause issues - // when the optimizer has restructured the query. - const isUserDefinedSubquery = queryMapping.has(from.query) - const subqueryFromAlias = getFirstFromAlias(from.query.from) - const isOptimizerCreated = - !isUserDefinedSubquery && from.alias === subqueryFromAlias - - if (!isOptimizerCreated) { - for (const [alias, whereClause] of subQueryResult.sourceWhereClauses) { - sourceWhereClauses.set(alias, whereClause) - } - } - - // Create a FLATTENED remapping from outer alias to innermost alias. - // For nested subqueries, this ensures one-hop lookups (not recursive chains). - // - // Example with 3-level nesting: - // Inner: .from({ user: usersCollection }) - // Middle: .from({ activeUser: innerSubquery }) → creates: activeUser → user - // Outer: .from({ author: middleSubquery }) → creates: author → user (not author → activeUser) - // - // The key insight: We search through the PULLED-UP aliasToCollectionId (which contains - // the innermost 'user' alias), so we always map directly to the deepest level. - // This means aliasRemapping[alias] is always a single lookup, never recursive. - // Needed for subscription resolution during lazy loading. - const innerAlias = Object.keys(subQueryResult.aliasToCollectionId).find( - (alias) => - subQueryResult.aliasToCollectionId[alias] === - subQueryResult.collectionId, - ) - if (innerAlias && innerAlias !== from.alias) { - aliasRemapping[from.alias] = innerAlias - } - - // Extract the pipeline from the compilation result - const subQueryInput = subQueryResult.pipeline - - // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY) - // We need to extract just the value for use in parent queries - const extractedInput = subQueryInput.pipe( - map((data: any) => { - const [ - key, - [value, _orderByIndex, correlationKey, parentContext, publicKey], - ] = data - // Unwrap Value expressions that might have leaked through as the entire row - const unwrapped = attachRouteMetadataToResult( - unwrapValue(value), - correlationKey, - parentContext, - publicKey, - ) - return [key, unwrapped] as [unknown, any] - }), - ) - - return { - alias: from.alias, - input: extractedInput, - collectionId: subQueryResult.collectionId, - sourceIncludes: - subQueryResult.includes?.map((include) => ({ - sourceAlias: from.alias, - include, - })) ?? [], - isParentRouted: parentKeyStream !== undefined, - } - } - default: - throw new UnsupportedFromTypeError((from as any).type) - } + const type = typeof key + const value = + type === `string` || type === `number` || type === `bigint` + ? String(key) + : JSON.stringify(key) + return `${type}:${value}` } // Only compiler-created Value instances are wrappers; user rows may have @@ -2101,7 +1954,9 @@ function getWholeRowSourceAliases(select: unknown): Array { ) } -function copySelectedRow(selected: Record) { +function copySelectedRow( + selected: Record, +): Record { return Array.isArray(selected) ? [...selected] : { ...selected } } @@ -2111,13 +1966,7 @@ function attachVirtualPropsToSelected( isCollectionRoot: boolean, metadataSources: ReadonlyArray | undefined, ): any { - if ( - !selected || - typeof selected !== `object` || - (!Array.isArray(selected) && !isPlainObject(selected)) - ) { - return selected - } + if (!isPublicContainer(selected)) return selected const selectedRecord = selected as Record // Union output is already a branch row, rather than an aliased source. @@ -2134,43 +1983,22 @@ function attachVirtualPropsToSelected( : undefined let result = selectedRecord for (const prop of VIRTUAL_PROP_NAMES) { - if (isCollectionRoot && prop in selectedRecord) { - // An explicit virtual name in a root projection cannot replace source - // metadata. Ordinary projections acquire their own Collection metadata. - const owner = - sourceRecord && prop in sourceRecord ? sourceRecord : undefined - if (owner) { - const value = owner[prop] - if (selectedRecord[prop] !== value) { - if (result === selectedRecord) - result = copySelectedRow(selectedRecord) - result[prop] = value - } - } else { - if (result === selectedRecord) result = copySelectedRow(selectedRecord) - delete result[prop] - } - } else if (selectedRecord[prop] == null && prop in row) { - if (result === selectedRecord) result = copySelectedRow(selectedRecord) - result[prop] = row[prop] - } + // A root projection cannot replace source metadata. Other projections + // only inherit a missing virtual property from their namespace. + const reset = isCollectionRoot && prop in selectedRecord + if (!reset && (selectedRecord[prop] != null || !(prop in row))) continue + const owner = reset ? sourceRecord : row + const present = owner !== undefined && prop in owner + const value = present ? owner[prop] : undefined + if (reset && present && selectedRecord[prop] === value) continue + if (result === selectedRecord) result = copySelectedRow(selectedRecord) + if (present) result[prop] = value + else delete result[prop] } return result } -function getIncludesPublicKey( - row: Record, - mainSource: string, - fallback: unknown, -): unknown { - return ( - row[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? - (row as any)[INCLUDES_PUBLIC_KEY] ?? - fallback - ) -} - /** * Recursively records the user-defined origin of optimized subqueries. * Compilation still consumes the optimized query; the mapping is lineage used @@ -2190,21 +2018,11 @@ function mapNestedQueries( i < optimizedQuery.join.length && i < originalQuery.join.length; i++ ) { - const optimizedJoin = optimizedQuery.join[i]! - const originalJoin = originalQuery.join[i]! - - if ( - optimizedJoin.from.type === `queryRef` && - originalJoin.from.type === `queryRef` - ) { - queryMapping.set(optimizedJoin.from.query, originalJoin.from.query) - // Recursively map nested queries in joins - mapNestedQueries( - optimizedJoin.from.query, - originalJoin.from.query, - queryMapping, - ) - } + mapNestedFromQueries( + optimizedQuery.join[i]!.from, + originalQuery.join[i]!.from, + queryMapping, + ) } } } @@ -2216,10 +2034,6 @@ function getAllSources(query: QueryIR): Array { ] } -function getFirstFromAlias(from: QueryIR[`from`]): string { - return getFromSources(from)[0]?.alias ?? `` -} - function findProjectedIncludePaths( select: Record, targetPath: Array, @@ -2280,24 +2094,9 @@ function findProjectedIncludePaths( } if (value instanceof ConditionalSelect) { - const previousBranchGuards: Array = [] - for (const branch of value.branches) { - visitSelectValue(branch.value, path, [ - ...guards, - ...previousBranchGuards, - { condition: branch.condition, expected: true }, - ]) - previousBranchGuards.push({ - condition: branch.condition, - expected: false, - }) - } - if (value.defaultValue !== undefined) { - visitSelectValue(value.defaultValue, path, [ - ...guards, - ...previousBranchGuards, - ]) - } + visitConditionalSelect(value, guards, (child, childGuards) => + visitSelectValue(child, path, childGuards), + ) return } @@ -2379,60 +2178,33 @@ function extractIncludesFromSelect(select: Record): Array<{ }> = [] for (const [key, value] of Object.entries(select)) { if (key.startsWith(`__SPREAD_SENTINEL__`)) continue - if (value instanceof IncludesSubquery) { - results.push({ - key: getIncludesRoutingKey([key], results), - path: [key], - subquery: value, - guards: [], - }) - } else if (value instanceof ConditionalSelect) { - collectIncludesFromConditionalSelect(value, [key], [], results) - } else if (isNestedSelectObject(value)) { - // Check nested objects for IncludesSubquery — not supported yet + if (isNestedSelectObject(value)) { + // Plain nested includes remain invalid; conditional branches may contain them. assertNoNestedIncludes(value, key) + } else { + collectIncludesFromSelectValue(value, [key], [], results) } } return results } -function collectIncludesFromConditionalSelect( +/** Each branch requires every earlier branch to be false. */ +function visitConditionalSelect( conditional: ConditionalSelect, - prefixPath: Array, guards: Array, - results: Array<{ - key: string - path: Array - subquery: IncludesSubquery - guards: Array - }>, + visit: (value: any, guards: Array) => void, ): void { const previousBranchGuards: Array = [] for (const branch of conditional.branches) { - collectIncludesFromSelectValue( - branch.value, - prefixPath, - [ - ...guards, - ...previousBranchGuards, - { condition: branch.condition, expected: true }, - ], - results, - ) - previousBranchGuards.push({ - condition: branch.condition, - expected: false, - }) - } - - if (conditional.defaultValue !== undefined) { - collectIncludesFromSelectValue( - conditional.defaultValue, - prefixPath, - [...guards, ...previousBranchGuards], - results, - ) + visit(branch.value, [ + ...guards, + ...previousBranchGuards, + { condition: branch.condition, expected: true }, + ]) + previousBranchGuards.push({ condition: branch.condition, expected: false }) } + if (conditional.defaultValue !== undefined) + visit(conditional.defaultValue, [...guards, ...previousBranchGuards]) } function collectIncludesFromSelectValue( @@ -2447,13 +2219,15 @@ function collectIncludesFromSelectValue( }>, ): void { if (value instanceof IncludesSubquery) { - const key = getIncludesRoutingKey(prefixPath, results) + const key = getUniqueIncludesRoutingKey(prefixPath.join(`.`), results) results.push({ key, path: prefixPath, subquery: value, guards }) return } if (value instanceof ConditionalSelect) { - collectIncludesFromConditionalSelect(value, prefixPath, guards, results) + visitConditionalSelect(value, guards, (child, childGuards) => + collectIncludesFromSelectValue(child, prefixPath, childGuards, results), + ) return } @@ -2467,13 +2241,6 @@ function collectIncludesFromSelectValue( } } -function getIncludesRoutingKey( - path: Array, - entries: Array<{ key: string }>, -): string { - return getUniqueIncludesRoutingKey(path.join(`.`), entries) -} - function getUniqueIncludesRoutingKey( baseKey: string, entries: Array<{ key?: string; fieldName?: string }>, @@ -2519,70 +2286,31 @@ function assertNoNestedIncludes( /** * Replaces the IncludesSubquery at `path` inside a select value, through * nested objects and conditional branches, with a null Value placeholder, so - * processSelect() never sees it. Returns the new value and whether anything - * was replaced, so each level rebuilds only the objects that changed. + * processSelect() never sees it. Unchanged values retain their identity, so + * each level rebuilds only the objects that changed. */ -function replaceIncludesInSelectValue( - value: any, - path: Array, -): { value: any; replaced: boolean } { - if (path.length === 0 && value instanceof IncludesSubquery) { - return { value: new ValClass(null), replaced: true } - } - +function replaceIncludesInSelectValue(value: any, path: Array): any { + if (path.length === 0 && value instanceof IncludesSubquery) + return new ValClass(null) if (value instanceof ConditionalSelect) { - return replaceIncludesInConditionalSelect(value, path) - } - - if (path.length === 0 || !isNestedSelectObject(value)) { - return { value, replaced: false } - } - + const branches = value.branches.map((branch) => { + const next = replaceIncludesInSelectValue(branch.value, path) + return Object.is(next, branch.value) ? branch : { ...branch, value: next } + }) + const defaultValue = + value.defaultValue === undefined + ? undefined + : replaceIncludesInSelectValue(value.defaultValue, path) + return branches.some((branch, index) => branch !== value.branches[index]) || + !Object.is(defaultValue, value.defaultValue) + ? new ConditionalSelect(branches, defaultValue) + : value + } + if (path.length === 0 || !isNestedSelectObject(value)) return value const [head, ...rest] = path - const result = replaceIncludesInSelectValue(value[head!], rest) - if (!result.replaced) { - return { value, replaced: false } - } - return { - value: { - ...value, - [head!]: result.value, - }, - replaced: true, - } -} - -function replaceIncludesInConditionalSelect( - conditional: ConditionalSelect, - path: Array, -): { value: ConditionalSelect; replaced: boolean } { - let replaced = false - const branches = conditional.branches.map((branch) => { - const result = replaceIncludesInSelectValue(branch.value, path) - if (!result.replaced) { - return branch - } - replaced = true - return { ...branch, value: result.value } - }) - - let defaultValue = conditional.defaultValue - if (conditional.defaultValue !== undefined) { - const result = replaceIncludesInSelectValue(conditional.defaultValue, path) - if (result.replaced) { - replaced = true - defaultValue = result.value - } - } - - if (!replaced) { - return { value: conditional, replaced: false } - } - - return { - value: new ConditionalSelect(branches, defaultValue), - replaced: true, - } + const child = value[head!] + const next = replaceIncludesInSelectValue(child, rest) + return Object.is(next, child) ? value : { ...value, [head!]: next } } /** diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 65f1e38092..494ab28604 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -12,16 +12,15 @@ import { InvalidJoinConditionSameSourceError, InvalidJoinConditionSourceMismatchError, JoinCollectionNotFoundError, + UnsupportedFromTypeError, UnsupportedJoinSourceTypeError, UnsupportedJoinTypeError, } from '../../errors.js' -import { - getParentContextIdentity, - getParentContextValue, -} from '../equality-value-identity.js' +import { getParentContextIdentity } from '../equality-value-identity.js' import { ensureIndexForField } from '../../indexes/auto-index.js' import { validateJoinConditions } from '../join-conditions.js' import { + Value as ValClass, collectPropRefs, getFromSources, getPropRefSourceAlias, @@ -32,17 +31,15 @@ import { getLazyLoadTargets } from './lazy-targets.js' import { crossJoinParentRoutes } from './parent-routes.js' import { queriesMatchForCaching } from './query-equivalence.js' import { - INCLUDES_PUBLIC_KEY, + CALLBACK_INPUT, attachRouteMetadata, attachRouteMetadataToResult, + getCallbackInput, getNamespacedRouteMetadata, - getRouteMetadata, - getRoutedScalarMetadata, - stripRouteMetadata, + wrapInputRow, } from './route-metadata.js' import type { ValueIdentity } from '../equality-value-identity.js' -import type { CompileQueryFn } from './index.js' -import type { OrderByOptimizationInfo } from './order-by.js' +import type { CompilationResult, IncludesCompilationResult } from './index.js' import type { BasicExpression, CollectionSourceRef, @@ -55,12 +52,17 @@ import type { import type { IStreamBuilder, JoinType } from '@tanstack/db-ivm' import type { Collection } from '../../collection/index.js' import type { + InputRow, KeyedStream, NamespacedAndKeyedStream, NamespacedRow, } from '../../types.js' -import type { QueryCache, QueryMapping, WindowOptions } from './types.js' -import type { CollectionSubscription } from '../../collection/subscription.js' +import type { QueryMapping } from './types.js' + +type CompileSubquery = ( + query: QueryIR, + parentKeyStream?: KeyedStream, +) => CompilationResult export type LazyDemandPlan = { id: string @@ -91,7 +93,7 @@ function parameterizeJoinInputByParentRoutes( return crossJoinParentRoutes( input, parentKeyStream, - (rowKey, row, correlationKey, parentContext) => { + (rowKey, row, correlationKey, parentContext, callbackValue) => { return [ serializeValue([ valueIdentity.equality(rowKey), @@ -105,42 +107,12 @@ function parameterizeJoinInputByParentRoutes( correlationKey, parentContext, ), - ] + callbackValue, + ] as unknown as [unknown, unknown] }, ) } -function wrapJoinedInputRow(alias: string, row: any): NamespacedRow { - const scalar = getRoutedScalarMetadata(row) - if (scalar) { - const namespaced = attachRouteMetadata( - { - ...getParentContextValue(scalar.parentContext), - [alias]: scalar.value, - [INCLUDES_PUBLIC_KEY]: scalar.publicKey, - }, - scalar.correlationKey, - scalar.parentContext, - ) as unknown as NamespacedRow - return namespaced - } - - if (row == null || typeof row !== `object`) { - return { [alias]: row } - } - - const route = getRouteMetadata(row) - const cleanRow = route ? stripRouteMetadata(row) : row - const namespaced: NamespacedRow = { - ...getParentContextValue(route?.parentContext), - [alias]: cleanRow, - } - if (route) { - attachRouteMetadata(namespaced, route.correlationKey, route.parentContext) - } - return namespaced -} - function getRouteJoinKey( row: NamespacedRow, source: string, @@ -218,17 +190,13 @@ export function processJoins( mainCollectionId: string, mainSource: string, allInputs: Record, - cache: QueryCache, queryMapping: QueryMapping, collections: Record, - subscriptions: Record, callbacks: Record, lazySources: Set, - optimizableOrderByCollections: Record, - setWindowFn: (windowFn: (options: WindowOptions) => void) => void, rawQuery: QueryIR, localFrom: From, - onCompileSubquery: CompileQueryFn, + onCompileSubquery: CompileSubquery, aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, @@ -236,308 +204,253 @@ export function processJoins( valueIdentity: ValueIdentity, parentKeyStream?: KeyedStream, ): NamespacedAndKeyedStream { - let resultPipeline = pipeline const localBindings = new Set( getFromSources(localFrom).map((source) => source.bindingId), ) - for (const joinClause of joinClauses) { - resultPipeline = processJoin( - resultPipeline, - joinClause, - sources, - mainCollectionId, - mainSource, + // The loop owns the source bindings and compiler context for each join. + function processJoin( + inputPipeline: NamespacedAndKeyedStream, + joinClause: JoinClause, + ): NamespacedAndKeyedStream { + const joinedSource = joinClause.from.alias + const mainAliases = Object.keys(sources) + const role = (ref: PropRef): JoinRefRole => { + const alias = getJoinRefAlias(ref) + if (ref.bindingId !== undefined) { + if (ref.bindingId === joinClause.from.bindingId) return `joined` + return localBindings.has(ref.bindingId) ? `main` : `ancestor` + } + if (alias === joinedSource) return `joined` + if ( + alias && + (mainAliases.includes(alias) || rawQuery.from.type === `unionAll`) + ) + return `main` + return `unknown` + } + const conditions = validateJoinConditions(joinClause.on).map( + ([left, right]) => + analyzeJoinExpressions(left, right, joinedSource, role), + ) + // The first equality supplies candidate demand; the full tuple decides matches. + const { mainExpr, joinedExpr } = conditions[0]! + const joinedExpressionUsesParent = conditions.some( + ({ joinedExpr: expression }) => + collectPropRefs(expression).some((ref) => { + const sourceRole = role(ref) + return sourceRole === `ancestor` || sourceRole === `unknown` + }), + ) + const routeJoinedSource = + parentKeyStream !== undefined && + (joinClause.from.type === `queryRef` || + ((joinClause.type === `right` || joinClause.type === `full`) && + !mainSourceIsParentFiltered) || + joinedExpressionUsesParent) + + // Get the joined source alias and input stream + const { + alias: processedJoinedSource, + input: joinedInput, + collectionId: joinedCollectionId, + } = processSource( + joinClause.from, allInputs, - cache, queryMapping, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - rawQuery, onCompileSubquery, aliasToCollectionId, aliasRemapping, sourceWhereClauses, - mainSourceIsParentFiltered, valueIdentity, - localBindings, - parentKeyStream, + `join`, + routeJoinedSource ? parentKeyStream : undefined, ) - localBindings.add(joinClause.from.bindingId) - } - - return resultPipeline -} -/** - * Processes a single join clause with lazy loading optimization. - * For LEFT/RIGHT/INNER joins, marks one side as "lazy" (loads on-demand based on join keys). - */ -function processJoin( - pipeline: NamespacedAndKeyedStream, - joinClause: JoinClause, - sources: Record, - mainCollectionId: string, - mainSource: string, - allInputs: Record, - cache: QueryCache, - queryMapping: QueryMapping, - collections: Record, - subscriptions: Record, - callbacks: Record, - lazySources: Set, - optimizableOrderByCollections: Record, - setWindowFn: (windowFn: (options: WindowOptions) => void) => void, - rawQuery: QueryIR, - onCompileSubquery: CompileQueryFn, - aliasToCollectionId: Record, - aliasRemapping: Record, - sourceWhereClauses: Map>, - mainSourceIsParentFiltered: boolean, - valueIdentity: ValueIdentity, - localBindings: ReadonlySet, - parentKeyStream?: KeyedStream, -): NamespacedAndKeyedStream { - const isCollectionRef = joinClause.from.type === `collectionRef` - - const joinedSource = joinClause.from.alias - const mainAliases = Object.keys(sources) - const role = (ref: PropRef): JoinRefRole => { - const alias = getJoinRefAlias(ref) - if (ref.bindingId !== undefined) { - if (ref.bindingId === joinClause.from.bindingId) return `joined` - return localBindings.has(ref.bindingId) ? `main` : `ancestor` + if (processedJoinedSource !== joinedSource) { + throw new InvalidJoinConditionSourceMismatchError() } - if (alias === joinedSource) return `joined` - if ( - alias && - (mainAliases.includes(alias) || rawQuery.from.type === `unionAll`) - ) - return `main` - return `unknown` - } - const conditions = validateJoinConditions(joinClause.on).map( - ([left, right]) => analyzeJoinExpressions(left, right, joinedSource, role), - ) - // The first equality supplies candidate demand; the full tuple decides matches. - const { mainExpr, joinedExpr } = conditions[0]! - const joinedExpressionUsesParent = conditions.some( - ({ joinedExpr: expression }) => - collectPropRefs(expression).some((ref) => { - const sourceRole = role(ref) - return sourceRole === `ancestor` || sourceRole === `unknown` - }), - ) - const routeJoinedSource = - parentKeyStream !== undefined && - (joinClause.from.type === `queryRef` || - ((joinClause.type === `right` || joinClause.type === `full`) && - !mainSourceIsParentFiltered) || - joinedExpressionUsesParent) - - // Get the joined source alias and input stream - const { - alias: processedJoinedSource, - input: joinedInput, - collectionId: joinedCollectionId, - } = processJoinSource( - joinClause.from, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - onCompileSubquery, - aliasToCollectionId, - aliasRemapping, - sourceWhereClauses, - valueIdentity, - routeJoinedSource ? parentKeyStream : undefined, - ) - if (processedJoinedSource !== joinedSource) { - throw new InvalidJoinConditionSourceMismatchError() - } - - // Add the joined source to the sources map - sources[joinedSource] = joinedInput - if (isCollectionRef) { - // Only direct collection references form new alias bindings. Subquery - // aliases reuse the mapping returned from the recursive compilation above. - aliasToCollectionId[joinedSource] = joinedCollectionId - } + // Add the joined source to the sources map + sources[joinedSource] = joinedInput - const mainCollection = collections[mainCollectionId] - const joinedCollection = collections[joinedCollectionId] + const mainCollection = collections[mainCollectionId] + const joinedCollection = collections[joinedCollectionId] - if (!mainCollection) { - throw new JoinCollectionNotFoundError(mainCollectionId) - } + if (!mainCollection) { + throw new JoinCollectionNotFoundError(mainCollectionId) + } - if (!joinedCollection) { - throw new JoinCollectionNotFoundError(joinedCollectionId) - } + if (!joinedCollection) { + throw new JoinCollectionNotFoundError(joinedCollectionId) + } - const sourceActivity = getActiveAndLazySources( - joinClause.type, - mainCollection, - joinedCollection, - mainSourceIsParentFiltered, - ) - const activeSource = routeJoinedSource - ? undefined - : sourceActivity.activeSource - const lazySource = sourceActivity.lazySource - - // Pre-compile the join expressions - const compiledMainExpressions = conditions.map((condition) => - compileExpression(condition.mainExpr), - ) - const compiledJoinedExpressions = conditions.map((condition) => - compileExpression(condition.joinedExpr), - ) + const sourceActivity = getActiveAndLazySources( + joinClause.type, + mainCollection, + joinedCollection, + mainSourceIsParentFiltered, + ) + const activeSource = routeJoinedSource + ? undefined + : sourceActivity.activeSource + const lazySource = sourceActivity.lazySource + + // Pre-compile the join expressions + const compiledMainExpressions = conditions.map((condition) => + compileExpression(condition.mainExpr), + ) + const compiledJoinedExpressions = conditions.map((condition) => + compileExpression(condition.joinedExpr), + ) - // Prepare both sides with the same key and raw-demand rules. - let mainPipeline = pipeline.pipe( - map(([currentKey, namespacedRow]) => - keyJoinInput( - currentKey, - namespacedRow, - mainSource, - `main`, - compiledMainExpressions, - routeJoinedSource, - valueIdentity, + // Prepare both sides with the same key and raw-demand rules. + let mainPipeline = inputPipeline.pipe( + map(([currentKey, namespacedRow]) => + keyJoinInput( + currentKey, + namespacedRow, + mainSource, + `main`, + compiledMainExpressions, + routeJoinedSource, + valueIdentity, + ), ), - ), - ) + ) - let joinedPipeline = joinedInput.pipe( - map(([currentKey, row]) => - keyJoinInput( - currentKey, - wrapJoinedInputRow(joinedSource, row), - joinedSource, - `joined`, - compiledJoinedExpressions, - routeJoinedSource, - valueIdentity, + let joinedPipeline = joinedInput.pipe( + map(([currentKey, row, callbackValue]: [...InputRow, unknown?]) => + keyJoinInput( + currentKey, + wrapInputRow(joinedSource, row, callbackValue), + joinedSource, + `joined`, + compiledJoinedExpressions, + routeJoinedSource, + valueIdentity, + ), ), - ), - ) - - // Apply the join operation - if (![`inner`, `left`, `right`, `full`].includes(joinClause.type)) { - throw new UnsupportedJoinTypeError(joinClause.type) - } - - if (activeSource) { - // If the lazy collection comes from a subquery that has a limit and/or an offset clause - // then we need to deoptimize the join because we don't know which rows are in the result set - // since we simply lookup matching keys in the index but the index contains all rows - // (not just the ones that pass the limit and offset clauses) - const lazyFrom = activeSource === `main` ? joinClause.from : rawQuery.from - const limitedSubquery = - lazyFrom.type === `queryRef` && - (lazyFrom.query.limit !== undefined || - lazyFrom.query.offset !== undefined || - lazyFrom.query.singleResult) - const resultUnionLazySide = lazyFrom.type === `unionAll` - - const lazySourceJoinExpr = activeSource === `main` ? joinedExpr : mainExpr - const lazyAlias = activeSource === `main` ? joinedSource : mainSource - const lazyTargets = resultUnionLazySide - ? [] - : getLazyLoadTargets( - rawQuery, - lazyFrom, - lazyAlias, - lazySourceJoinExpr, - lazySource, - aliasRemapping, - ) + ) - if (!limitedSubquery && lazyTargets.length > 0) { - // This join can be optimized by having the active collection - // dynamically load keys into the lazy collection - // based on the value of the joinKey and by looking up - // matching rows in the index of the lazy collection - - // Mark the lazy source aliases as lazy - // this Set is passed by the liveQueryCollection to the compiler - // such that the liveQueryCollection can check it after compilation - // to know which source aliases should load data lazily (not initially) - for (const target of lazyTargets) { - lazySources.add(target.sourceId) - } + // Apply the join operation + if (![`inner`, `left`, `right`, `full`].includes(joinClause.type)) { + throw new UnsupportedJoinTypeError(joinClause.type) + } - const demandPlans = lazyTargets.map((target) => - registerLazyDemandPlan(callbacks, target), - ) - const demandWeights = new Map() + if (activeSource) { + // If the lazy collection comes from a subquery that has a limit and/or an offset clause + // then we need to deoptimize the join because we don't know which rows are in the result set + // since we simply lookup matching keys in the index but the index contains all rows + // (not just the ones that pass the limit and offset clauses) + const lazyFrom = activeSource === `main` ? joinClause.from : rawQuery.from + const limitedSubquery = + lazyFrom.type === `queryRef` && + (lazyFrom.query.limit !== undefined || + lazyFrom.query.offset !== undefined || + lazyFrom.query.singleResult) + const resultUnionLazySide = lazyFrom.type === `unionAll` + + const lazySourceJoinExpr = activeSource === `main` ? joinedExpr : mainExpr + const lazyAlias = activeSource === `main` ? joinedSource : mainSource + const lazyTargets = resultUnionLazySide + ? [] + : getLazyLoadTargets( + rawQuery, + lazyFrom, + lazyAlias, + lazySourceJoinExpr, + lazySource, + aliasRemapping, + ) - const activePipeline = - activeSource === `main` ? mainPipeline : joinedPipeline + if (!limitedSubquery && lazyTargets.length > 0) { + // This join can be optimized by having the active collection + // dynamically load keys into the lazy collection + // based on the value of the joinKey and by looking up + // matching rows in the index of the lazy collection + + // Mark the lazy source aliases as lazy + // this Set is passed by the liveQueryCollection to the compiler + // such that the liveQueryCollection can check it after compilation + // to know which source aliases should load data lazily (not initially) + for (const target of lazyTargets) { + lazySources.add(target.sourceId) + } - for (const target of lazyTargets) { - const fieldName = target.path[0] - if (fieldName) { - ensureIndexForField(fieldName, target.path, target.collection) + const demandPlans = lazyTargets.map((target) => + registerLazyDemandPlan(callbacks, target), + ) + const demandWeights = new Map< + string, + { key: unknown; weight: number } + >() + + const activePipeline = + activeSource === `main` ? mainPipeline : joinedPipeline + + for (const target of lazyTargets) { + const fieldName = target.path[0] + if (fieldName) { + ensureIndexForField(fieldName, target.path, target.collection) + } } - } - // Set up lazy loading: intercept active side's stream and dynamically load - // matching rows from lazy side based on join keys. - const activePipelineWithLoading: IStreamBuilder< - [key: string, value: JoinInputValue] - > = activePipeline.pipe( - tap((data) => { - for (const [[joinKey, [, , joinValue]], weight] of data.getInner()) { - if (joinValue == null) continue - const previous = demandWeights.get(joinKey) - const nextWeight = (previous?.weight ?? 0) + weight - if (nextWeight === 0) { - demandWeights.delete(joinKey) - } else { - demandWeights.set(joinKey, { - key: previous?.key ?? joinValue, - weight: nextWeight, - }) + // Set up lazy loading: intercept active side's stream and dynamically load + // matching rows from lazy side based on join keys. + const activePipelineWithLoading: IStreamBuilder< + [key: string, value: JoinInputValue] + > = activePipeline.pipe( + tap((data) => { + for (const [ + [joinKey, [, , joinValue]], + weight, + ] of data.getInner()) { + if (joinValue == null) continue + const previous = demandWeights.get(joinKey) + const nextWeight = (previous?.weight ?? 0) + weight + if (nextWeight === 0) { + demandWeights.delete(joinKey) + } else { + demandWeights.set(joinKey, { + key: previous?.key ?? joinValue, + weight: nextWeight, + }) + } } - } - const keys = new Set( - [...demandWeights.values()] - .filter(({ weight }) => weight > 0) - .map(({ key }) => key), - ) - for (let index = 0; index < lazyTargets.length; index++) { - const target = lazyTargets[index]! - callbacks[target.sourceId]?.setDemand?.(demandPlans[index]!, keys) - } - }), - ) + const keys = new Set( + [...demandWeights.values()] + .filter(({ weight }) => weight > 0) + .map(({ key }) => key), + ) + for (let index = 0; index < lazyTargets.length; index++) { + const target = lazyTargets[index]! + callbacks[target.sourceId]?.setDemand?.(demandPlans[index]!, keys) + } + }), + ) - if (activeSource === `main`) { - mainPipeline = activePipelineWithLoading - } else { - joinedPipeline = activePipelineWithLoading + if (activeSource === `main`) { + mainPipeline = activePipelineWithLoading + } else { + joinedPipeline = activePipelineWithLoading + } } } + + return mainPipeline.pipe( + joinOperator(joinedPipeline, joinClause.type as JoinType), + processJoinResults(joinedSource, mainAliases), + ) } - return mainPipeline.pipe( - joinOperator(joinedPipeline, joinClause.type as JoinType), - processJoinResults(joinedSource, mainAliases), - ) + for (const joinClause of joinClauses) { + pipeline = processJoin(pipeline, joinClause) + localBindings.add(joinClause.from.bindingId) + } + + return pipeline } type JoinRefRole = `joined` | `main` | `ancestor` | `unknown` @@ -613,26 +526,25 @@ function analyzeJoinExpressions( } /** - * Processes the join source (collection or sub-query) + * Compiles a FROM or JOIN source and propagates its source metadata. */ -function processJoinSource( +export function processSource( from: CollectionSourceRef | QueryRef, allInputs: Record, - collections: Record, - subscriptions: Record, - callbacks: Record, - lazySources: Set, - optimizableOrderByCollections: Record, - setWindowFn: (windowFn: (options: WindowOptions) => void) => void, - cache: QueryCache, queryMapping: QueryMapping, - onCompileSubquery: CompileQueryFn, + onCompileSubquery: CompileSubquery, aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, valueIdentity: ValueIdentity, + sourceKind: `from` | `join`, parentKeyStream?: KeyedStream, -): { alias: string; input: KeyedStream; collectionId: string } { +): { + alias: string + input: KeyedStream + collectionId: string + includes?: Array +} { switch (from.type) { case `descriptorRef`: return requireCollectionSource(from) @@ -648,13 +560,14 @@ function processJoinSource( aliasToCollectionId[from.alias] = from.collection.id return { alias: from.alias, - input: parentKeyStream - ? parameterizeJoinInputByParentRoutes( - input, - parentKeyStream, - valueIdentity, - ) - : input, + input: + sourceKind === `join` && parentKeyStream + ? parameterizeJoinInputByParentRoutes( + input, + parentKeyStream, + valueIdentity, + ) + : input, collectionId: from.collection.id, } } @@ -667,67 +580,27 @@ function processJoinSource( originalQuery && queriesMatchForCaching(from.query, originalQuery) ? originalQuery : from.query - const subQueryResult = onCompileSubquery( - queryToCompile, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - parentKeyStream, - ) + const subQueryResult = onCompileSubquery(queryToCompile, parentKeyStream) - // Pull up alias mappings from subquery to parent scope. - // This includes both the innermost alias-to-collection mappings AND - // any existing remappings from nested subquery levels. Object.assign(aliasToCollectionId, subQueryResult.aliasToCollectionId) Object.assign(aliasRemapping, subQueryResult.aliasRemapping) - - // Pull up source WHERE clauses from subquery to parent scope. - // This enables loadSubset to receive the correct where clauses for subquery collections. - // - // IMPORTANT: Skip pull-up for optimizer-created subqueries. These are detected when: - // 1. The outer alias (from.alias) matches the inner alias (from.query.from.alias) - // 2. The subquery was found in queryMapping (it's a user-defined subquery, not optimizer-created) - // - // For optimizer-created subqueries, the parent already has the sourceWhereClauses - // extracted from the original raw query, so pulling up would be redundant. - const isUserDefinedSubquery = queryMapping.has(from.query) - const fromInnerAlias = getFirstFromAlias(from.query) - const isOptimizerCreated = - !isUserDefinedSubquery && - fromInnerAlias !== undefined && - from.alias === fromInnerAlias - - if (!isOptimizerCreated) { - for (const [alias, whereClause] of subQueryResult.sourceWhereClauses) { - sourceWhereClauses.set(alias, whereClause) - } + // Optimizer-created wrappers already contributed their predicates to the + // parent. User-authored subqueries must export their own source predicates. + if ( + queryMapping.has(from.query) || + from.alias !== getFromSources(from.query.from)[0]?.alias + ) { + for (const [alias, where] of subQueryResult.sourceWhereClauses) + sourceWhereClauses.set(alias, where) } - - // Create a flattened remapping from outer alias to innermost alias. - // For nested subqueries, this ensures one-hop lookups (not recursive chains). - // - // Example with 3-level nesting: - // Inner: .from({ user: usersCollection }) - // Middle: .from({ activeUser: innerSubquery }) → creates: activeUser → user - // Outer: .join({ author: middleSubquery }, ...) → creates: author → user (not author → activeUser) - // - // We search through the PULLED-UP aliasToCollectionId (which contains the - // innermost 'user' alias), so we always map directly to the deepest level. - // This means aliasRemapping[lazyAlias] is always a single lookup, never recursive. + // Resolve directly to the innermost alias, so nested scopes need one lookup. const innerAlias = Object.keys(subQueryResult.aliasToCollectionId).find( (alias) => subQueryResult.aliasToCollectionId[alias] === subQueryResult.collectionId, ) - if (innerAlias && innerAlias !== from.alias) { + if (innerAlias && innerAlias !== from.alias) aliasRemapping[from.alias] = innerAlias - } // Extract the pipeline from the compilation result const subQueryInput = subQueryResult.pipeline @@ -738,20 +611,30 @@ function processJoinSource( map((data: any) => { const [ key, - [value, _orderByIndex, correlationKey, parentContext, publicKey], + [ + value, + _orderByIndex, + correlationKey, + parentContext, + publicKey, + callbackValue, + ], ] = data - if (!parentKeyStream) { - return [key, value] as [unknown, any] + if (sourceKind === `join` && !parentKeyStream) { + return [key, value, callbackValue] as unknown as InputRow } return [ key, attachRouteMetadataToResult( - value, + sourceKind === `from` && value instanceof ValClass + ? value.value + : value, correlationKey, parentContext, publicKey, ), - ] as [unknown, any] + callbackValue, + ] as unknown as InputRow }), ) @@ -759,10 +642,13 @@ function processJoinSource( alias: from.alias, input: extractedInput as KeyedStream, collectionId: subQueryResult.collectionId, + includes: subQueryResult.includes, } } default: - throw new UnsupportedJoinSourceTypeError((from as any).type) + throw sourceKind === `join` + ? new UnsupportedJoinSourceTypeError((from as any).type) + : new UnsupportedFromTypeError((from as any).type) } } @@ -774,10 +660,6 @@ function encodeNonFiniteKey(_: string, value: unknown): unknown { : value } -function getFirstFromAlias(query: QueryIR): string | undefined { - return getFromSources(query.from)[0]?.alias -} - function processJoinResults( joinedSource: string, mainAliases: ReadonlyArray, @@ -795,22 +677,33 @@ function processJoinResults( const joinedKey = joined?.[0] const joinedNamespacedRow = joined?.[1] - // A joined row carries inherited parent aliases. When the main row is - // present it owns that scope; copy only the joined source from its row. - const mergedNamespacedRow: NamespacedRow = Object.assign( - {}, - mainNamespacedRow ?? joinedNamespacedRow, - ) - if (main == null) { - // The joined row may carry parent context with a local main alias. - // An absent main side must not turn that ancestor into a child row. - for (const alias of mainAliases) delete mergedNamespacedRow[alias] + // The main side owns inherited parent aliases. Missing local sides + // must not expose an ancestor with the same alias. + const mergeRows = ( + mainRow: NamespacedRow | undefined, + joinedRow: NamespacedRow | undefined, + ): NamespacedRow => { + const row = { ...(mainRow ?? joinedRow) } as NamespacedRow + if (main == null) for (const alias of mainAliases) delete row[alias] + if (joinedRow) row[joinedSource] = joinedRow[joinedSource]! + else delete row[joinedSource] + return row } - if (joinedNamespacedRow) { - mergedNamespacedRow[joinedSource] = joinedNamespacedRow[joinedSource]! - } else { - // A missing joined side cannot inherit a same-named parent source. - delete mergedNamespacedRow[joinedSource] + const mergedNamespacedRow = mergeRows( + mainNamespacedRow, + joinedNamespacedRow, + ) + const mainView = + mainNamespacedRow && getCallbackInput(mainNamespacedRow) + const joinedView = + joinedNamespacedRow && getCallbackInput(joinedNamespacedRow) + if ( + mainView !== mainNamespacedRow || + joinedView !== joinedNamespacedRow + ) { + Object.assign(mergedNamespacedRow, { + [CALLBACK_INPUT]: mergeRows(mainView, joinedView), + }) } // Combine the main and joined keys without ambiguity: keys may contain diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index 518a0fddad..01c61fbf48 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -17,6 +17,7 @@ import { supportsOrderedRange, } from '../../utils/index-optimization.js' import { hasVirtualPropPath } from '../../virtual-props.js' +import { deepEquals } from '../../utils.js' import { devBuild } from '../../error-message.js' import { compileExpression } from './evaluators.js' import { getSourceAliasesFromExpression } from './expressions.js' @@ -66,8 +67,8 @@ export type OrderByOptimizationInfo = { requiresFullSource: boolean /** Whether the ordered source's installed rows are the whole source. */ sourceHoldsAllRows: boolean - /** Source whose lazy demand must settle before a joined-filter page. */ - joinedFilterSourceId?: string + /** Sources whose lazy demand must settle before a joined-filter page. */ + joinedFilterSourceIds?: ReadonlyArray } function warnOrderIndexFallback( @@ -146,7 +147,7 @@ export function processOrderBy( return { compiledExpression: compileExpression(clauseWithoutAggregates), - compareOptions: buildCompareOptions(clause, collection), + compare: makeComparator(buildCompareOptions(clause, collection)), } }) // The row holds source aliases and any projected aggregate under $selected. @@ -159,30 +160,19 @@ export function processOrderBy( : null } - // Compare order terms in declaration order, stopping at the first difference. - const compare = (a: unknown, b: unknown) => { - if (orderByClause.length > 1) { - const arrayA = a as Array - const arrayB = b as Array - for (let i = 0; i < orderByClause.length; i++) { - const clause = compiledOrderBy[i]! - const compareFn = makeComparator(clause.compareOptions) - const result = compareFn(arrayA[i], arrayB[i]) - if (result !== 0) { - return result + // Compile comparators once; each row comparison reads the same options. + const compare = + compiledOrderBy.length <= 1 + ? (compiledOrderBy[0]?.compare ?? defaultComparator) + : (a: unknown, b: unknown) => { + const arrayA = a as Array + const arrayB = b as Array + for (let i = 0; i < compiledOrderBy.length; i++) { + const result = compiledOrderBy[i]!.compare(arrayA[i], arrayB[i]) + if (result !== 0) return result + } + return arrayA.length - arrayB.length } - } - return arrayA.length - arrayB.length - } - - if (orderByClause.length === 1) { - const clause = compiledOrderBy[0]! - const compareFn = makeComparator(clause.compareOptions) - return compareFn(a, b) - } - - return defaultComparator(a, b) - } let setSizeCallback: SetSizeCallback | undefined @@ -304,7 +294,9 @@ export function processOrderBy( valueExtractorForRawRow: extract, index, orderBy: sourceOrderBy, - joinedFilterSourceId, + joinedFilterSourceIds: joinedFilterSourceId + ? [joinedFilterSourceId] + : undefined, sourceHoldsAllRows: followRefResult.collection.config.syncMode !== `on-demand`, requiresFullSource: @@ -332,12 +324,48 @@ export function processOrderBy( // Ordered loading is owned by one lexical source. A collection can occur // more than once in a query tree, so collection ID and alias are not // sufficient identities here. - optimizableOrderByCollections[orderBySourceId] = info + const previous = optimizableOrderByCollections[orderBySourceId] + // One lexical source can feed separate authored/source metadata views. + // Each top-K owns its own window. Acquisition must cover all windows, + // including moves made through a previously compiled root's callback. + // Keep these reads live: setWindowFn updates the individual info objects. + optimizableOrderByCollections[orderBySourceId] = previous + ? { + ...info, + joinedFilterSourceIds: [ + ...new Set([ + ...(previous.joinedFilterSourceIds ?? []), + ...(info.joinedFilterSourceIds ?? []), + ]), + ], + get offset() { + return Math.min(previous.offset, info.offset) + }, + get limit() { + return ( + Math.max( + previous.offset + previous.limit, + info.offset + info.limit, + ) - this.offset + ) + }, + // Plan shape is fixed at compilation; only window bounds move. + requiresFullSource: + !deepEquals(previous.orderBy, info.orderBy) || + previous.requiresFullSource || + info.requiresFullSource, + dataNeeded: (boundary) => + Math.max( + previous.dataNeeded?.(boundary) ?? 0, + info.dataNeeded?.(boundary) ?? 0, + ), + } + : info // loadMoreIfNeeded reads how many rows the window needs. Without an // index, only an on-demand joined filter can leave the window short. const needsCount = - info.joinedFilterSourceId !== undefined && !info.sourceHoldsAllRows + joinedFilterSourceId !== undefined && !info.sourceHoldsAllRows if (index || needsCount) { setSizeCallback = (getSize) => { info.dataNeeded = (boundary) => { @@ -358,59 +386,31 @@ export function processOrderBy( } } - // Use grouped ordering when a groupKeyFn is provided (includes with limit/offset), - // otherwise use the standard global ordering operator. - if (groupKeyFn) { - return pipeline.pipe( - groupedOrderByWithFractionalIndex(valueExtractor, { - limit, - offset, - comparator: compare, - setSizeCallback, - groupKeyFn, - setWindowFn: ( - windowFn: (options: { offset?: number; limit?: number }) => void, - ) => { - setWindowFn((options) => { - windowFn(options) - if (orderByOptimizationInfo) { - orderByOptimizationInfo.offset = - options.offset ?? orderByOptimizationInfo.offset - orderByOptimizationInfo.limit = - options.limit ?? orderByOptimizationInfo.limit - } - }) - }, - }), - ) + const options = { + limit, + offset, + comparator: compare, + setSizeCallback, + setWindowFn: (windowFn: (options: WindowOptions) => void) => { + // Keep demand calculations aligned with the operator's current window. + setWindowFn((window) => { + windowFn(window) + if (orderByOptimizationInfo) { + orderByOptimizationInfo.offset = + window.offset ?? orderByOptimizationInfo.offset + orderByOptimizationInfo.limit = + window.limit ?? orderByOptimizationInfo.limit + } + }) + }, } - - // Use fractional indexing and return the tuple [value, index] return pipeline.pipe( - orderByWithFractionalIndex(valueExtractor, { - limit, - offset, - comparator: compare, - setSizeCallback, - setWindowFn: ( - windowFn: (options: { offset?: number; limit?: number }) => void, - ) => { - setWindowFn( - // We wrap the move function such that we update the orderByOptimizationInfo - // because that is used by the `dataNeeded` callback to determine if we need to load more data - (options) => { - windowFn(options) - if (orderByOptimizationInfo) { - orderByOptimizationInfo.offset = - options.offset ?? orderByOptimizationInfo.offset - orderByOptimizationInfo.limit = - options.limit ?? orderByOptimizationInfo.limit - } - }, - ) - }, - }), - // orderByWithFractionalIndex returns [key, [value, index]] - we keep this format + groupKeyFn + ? groupedOrderByWithFractionalIndex(valueExtractor, { + ...options, + groupKeyFn, + }) + : orderByWithFractionalIndex(valueExtractor, options), ) } diff --git a/packages/db/src/query/compiler/parent-routes.ts b/packages/db/src/query/compiler/parent-routes.ts index b6eddb32f9..1a28b7c2af 100644 --- a/packages/db/src/query/compiler/parent-routes.ts +++ b/packages/db/src/query/compiler/parent-routes.ts @@ -11,13 +11,17 @@ export function crossJoinParentRoutes( row: unknown, correlationKey: unknown, parentContext: unknown, + callbackValue?: unknown, ) => [unknown, unknown], ): KeyedStream { // Recursive sources need their route before a correlation field is always // available. The constant key intentionally creates one copy of each input // row per active route; callers filter those copies once the field is visible. const rows: any = input.pipe( - map(([rowKey, row]) => [PARENT_ROUTE_CROSS_KEY, [rowKey, row]]), + map(([rowKey, row, callbackValue]: [unknown, unknown, unknown?]) => [ + PARENT_ROUTE_CROSS_KEY, + [rowKey, row, callbackValue], + ]), ) const routes: any = parentKeyStream.pipe( map(([correlationKey, parentContext]) => [ @@ -32,9 +36,9 @@ export function crossJoinParentRoutes( Boolean(rowSide != null && routeSide != null), ), map(([, [rowSide, routeSide]]: any) => { - const [rowKey, row] = rowSide + const [rowKey, row, callbackValue] = rowSide const [correlationKey, parentContext] = routeSide - return assemble(rowKey, row, correlationKey, parentContext) + return assemble(rowKey, row, correlationKey, parentContext, callbackValue) }), ) as KeyedStream } diff --git a/packages/db/src/query/compiler/route-metadata.ts b/packages/db/src/query/compiler/route-metadata.ts index 9a5fad30d0..25a9f0a710 100644 --- a/packages/db/src/query/compiler/route-metadata.ts +++ b/packages/db/src/query/compiler/route-metadata.ts @@ -1,12 +1,18 @@ import { isPlainObject } from '../../utils/type-guards.js' +import { getParentContextValue } from '../equality-value-identity.js' +import type { NamespacedRow } from '../../types.js' const ROUTED_SCALAR_VALUE = Symbol(`tanstack_db_routed_scalar_value`) +export const CALLBACK_INPUT = Symbol(`tanstack_db_callback_input`) +export const CALLBACK_SELECTED = Symbol(`tanstack_db_callback_selected`) const ROUTE_METADATA = Symbol(`tanstack_db_route_metadata`) export const INCLUDES_PUBLIC_KEY = Symbol(`includesPublicKey`) export const INCLUDES_ROUTING = Symbol(`includesRouting`) const INTERNAL_ROUTE_KEYS = new Set([ ROUTE_METADATA, INCLUDES_PUBLIC_KEY, + CALLBACK_INPUT, + CALLBACK_SELECTED, ]) const INTERNAL_CALLBACK_KEYS = new Set([ ...INTERNAL_ROUTE_KEYS, @@ -19,9 +25,15 @@ type RoutedResult = { [INCLUDES_PUBLIC_KEY]: unknown } -type PublicContainerProperty = { - descriptor: PropertyDescriptor - value?: { original: unknown; replacement: unknown } +type PublicContainer = { + value: object + properties: Record< + PropertyKey, + PropertyDescriptor & { child?: PublicContainer } + > + parents: Array + dirty?: true + copy?: object } export type RouteMetadata = { @@ -89,19 +101,13 @@ export function attachRouteMetadataToResult( return value } - if (isPlainObject(value)) { - return { - ...value, + return Object.assign( + isPlainObject(value) ? { ...value } : { [ROUTED_SCALAR_VALUE]: value }, + { [ROUTE_METADATA]: { correlationKey, parentContext }, [INCLUDES_PUBLIC_KEY]: publicKey, - } - } - - return { - [ROUTED_SCALAR_VALUE]: value, - [ROUTE_METADATA]: { correlationKey, parentContext }, - [INCLUDES_PUBLIC_KEY]: publicKey, - } satisfies RoutedResult + }, + ) } export function getRoutedScalarMetadata( @@ -116,15 +122,52 @@ export function getRoutedScalarMetadata( } const routed = value as RoutedResult - const route = routed[ROUTE_METADATA] return { value: routed[ROUTED_SCALAR_VALUE], - correlationKey: route.correlationKey, - parentContext: route.parentContext, + ...routed[ROUTE_METADATA], publicKey: routed[INCLUDES_PUBLIC_KEY], } } +/** Give FROM and JOIN rows the same route and callback namespace. */ +export function wrapInputRow( + alias: string, + row: unknown, + callbackValue?: unknown, +): NamespacedRow { + const scalar = getRoutedScalarMetadata(row) + const route = scalar ?? getRouteMetadata(row) + const namespaced = { + ...getParentContextValue(route?.parentContext), + [alias]: scalar + ? scalar.value + : route + ? stripRouteMetadata(row as Record) + : row, + } as NamespacedRow + if (route) + attachRouteMetadata(namespaced, route.correlationKey, route.parentContext) + if (scalar) + Object.assign(namespaced, { [INCLUDES_PUBLIC_KEY]: scalar.publicKey }) + if (callbackValue !== undefined && callbackValue !== row) + Object.assign(namespaced, { [CALLBACK_INPUT]: { [alias]: callbackValue } }) + return namespaced +} + +/** A namespace keeps authored values beside its callback source overrides. + * A one-element tuple denotes a flat union view: absent fields must disappear, + * whereas named overrides replace only their own source aliases. + */ +export function getCallbackInput(row: T): T { + const view = (row as Record)[CALLBACK_INPUT] + if (Array.isArray(view)) { + const result = { ...view[0] } + if (`$selected` in row) result.$selected = row.$selected + return result as T + } + return view && typeof view === `object` ? { ...row, ...view } : row +} + /** Copy public containers while removing private route state at every depth. */ export function stripInternalRouteMetadata(value: unknown): unknown { return transformPublicContainers(value, (leaf) => leaf, INTERNAL_ROUTE_KEYS) @@ -149,80 +192,60 @@ export function transformPublicContainers( if (!Object.is(rootReplacement, value)) return rootReplacement if (!isPublicContainer(value)) return value - const parents = new WeakMap>() - const properties = new WeakMap< - object, - Map - >() - const dirty = new Set() - const visit = (current: object): void => { - if (properties.has(current)) return - const currentProperties = new Map() - properties.set(current, currentProperties) - for (const key of Reflect.ownKeys(current)) { + const containers = new Map() + const queue: Array = [] + const markDirty = (container: PublicContainer) => { + if (container.dirty) return + container.dirty = true + queue.push(container) + } + const visit = (current: object): PublicContainer => { + const existing = containers.get(current) + if (existing) return existing + const container: PublicContainer = { + value: current, + properties: Object.getOwnPropertyDescriptors(current), + parents: [], + } + containers.set(current, container) + for (const key of Reflect.ownKeys(container.properties)) { if (omittedKeys.has(key)) { - dirty.add(current) + delete container.properties[key] + markDirty(container) continue } - const descriptor = Object.getOwnPropertyDescriptor(current, key) - if (!descriptor) continue - const property: PublicContainerProperty = { descriptor } - currentProperties.set(key, property) + const descriptor = container.properties[key]! if (!descriptor.enumerable || !(`value` in descriptor)) continue const child = descriptor.value - const replacement = transformLeaf(child) - property.value = { original: child, replacement } - if (!Object.is(replacement, child)) { - dirty.add(current) - continue + descriptor.value = transformLeaf(child) + if (!Object.is(descriptor.value, child)) markDirty(container) + else if (isPublicContainer(child)) { + descriptor.child = visit(child) + descriptor.child.parents.push(container) } - if (!isPublicContainer(child)) continue - const childParents = parents.get(child) ?? new Set() - childParents.add(current) - parents.set(child, childParents) - visit(child) } + return container } - visit(value) - - const queue = [...dirty] - for (const current of queue) { - for (const parent of parents.get(current) ?? []) { - if (dirty.has(parent)) continue - dirty.add(parent) - queue.push(parent) - } + const root = visit(value) + for (const container of queue) { + for (const parent of container.parents) markDirty(parent) } - if (!dirty.has(value)) return value - - const copies = new WeakMap() - const copy = (current: object): object => { - if (!dirty.has(current)) return current - const existing = copies.get(current) - if (existing) return existing - - const result = Array.isArray(current) + const copy = (container: PublicContainer): object => { + if (!container.dirty) return container.value + if (container.copy) return container.copy + const result = Array.isArray(container.value) ? [] - : Object.create(Object.getPrototypeOf(current)) - copies.set(current, result) - for (const [key, property] of properties.get(current) ?? []) { - const descriptor = { ...property.descriptor } - if (property.value) { - const { original, replacement } = property.value - descriptor.value = !Object.is(replacement, original) - ? replacement - : isPublicContainer(original) - ? copy(original) - : original - } - Object.defineProperty(result, key, descriptor) + : Object.create(Object.getPrototypeOf(container.value)) + container.copy = result + for (const key of Reflect.ownKeys(container.properties)) { + const descriptor = container.properties[key]! + if (descriptor.child) descriptor.value = copy(descriptor.child) } - return result + return Object.defineProperties(result, container.properties) } - - return copy(value) + return copy(root) } -function isPublicContainer(value: unknown): value is object { +export function isPublicContainer(value: unknown): value is object { return Array.isArray(value) || isPlainObject(value) } diff --git a/packages/db/src/query/compiler/types.ts b/packages/db/src/query/compiler/types.ts index 83992e512d..f20a089bef 100644 --- a/packages/db/src/query/compiler/types.ts +++ b/packages/db/src/query/compiler/types.ts @@ -4,7 +4,10 @@ import type { CompilationResult } from './index.js' /** * Cache for compiled subqueries to avoid duplicate compilation */ -export type QueryCache = WeakMap +export type QueryCache = WeakMap< + QueryIR, + [authored?: CompilationResult, source?: CompilationResult] +> /** * Lineage from optimized queries back to their user-defined queries. diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 4355c7a753..9f1586211d 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -717,8 +717,8 @@ class EffectPipelineRunner { this.unsubscribeCallbacks.add(unsubscribe) if ( - Object.values(this.optimizableOrderByCollections).some( - (info) => info.joinedFilterSourceId === sourceId, + Object.values(this.optimizableOrderByCollections).some((info) => + info.joinedFilterSourceIds?.includes(sourceId), ) ) { this.unsubscribeCallbacks.add( @@ -764,8 +764,9 @@ class EffectPipelineRunner { () => false, undefined, () => - orderByInfo.joinedFilterSourceId !== undefined && - this.hasPendingJoinedWork(orderByInfo.joinedFilterSourceId), + orderByInfo.joinedFilterSourceIds?.some((joinedSourceId) => + this.hasPendingJoinedWork(joinedSourceId), + ) ?? false, ) this.orderedLoaders.set(sourceId, loader) loader.start() @@ -1143,10 +1144,10 @@ class EffectPipelineRunner { } private hasPendingJoinedFilterWork(): boolean { - return Object.values(this.optimizableOrderByCollections).some( - (info) => - info.joinedFilterSourceId !== undefined && - this.hasPendingJoinedWork(info.joinedFilterSourceId), + return Object.values(this.optimizableOrderByCollections).some((info) => + info.joinedFilterSourceIds?.some((sourceId) => + this.hasPendingJoinedWork(sourceId), + ), ) } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 6dc031ae33..158d40dcf6 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -386,22 +386,52 @@ internal matches. The public-container copy matrix crosses reference-key type, ordered and unordered subqueries, materialization form, and parent/child updates. Reserved public row fields have a separate callback-input law (maintainer -decision, 2026-10-28). Direct source rows and structured whole-row source +decisions, 2026-10-28 and 2026-10-30). Direct source rows and structured whole-row source spreads expose genuine source `$key`, `$origin`, `$hasPendingWrites`, and `$collectionId`, including when a spread selects conflicting reserved names. A subquery projection that creates its own row shape, including a new functional object or functional spread, exposes none of those reserved fields -before downstream `fn.select()` code runs. An inline include value keeps its +before downstream `fn.select()`, `fn.where()`, or `fn.having()` code runs. +The October 30 decision extends the original selection-only promise to both +functional predicates. An inline include value keeps its authored selected shape. Sanitizing only the eventual Collection publication cannot satisfy this law, because user code may already have derived an ordinary field from a spoofed input. -Known implementation gap for the first RC: correlated-child functional callback -inputs can still expose reserved fields from a new shape or authored collisions -from a structured source spread. The root callback receiving path passes its -bounded checks. The callback-input law above remains accepted; final publication -sanitization does not close this earlier boundary. The separate runtime/type -owners and exact RED witnesses are preserved in the +The compiler carries a callback view beside each authored selected-row +occurrence. FROM, joined-source and union adapters transport that view. +Functional callbacks read it without rewriting the query's authored row. +Private callback state never crosses the callback or result boundary. +An inline-only QueryRef and an inline child's final authored value retain +that authored shape, including when a predicate runs without a selection. +The selected-row normalization flag describes the output metadata view and +is independent of callback-view transport. A two-slot cache retains authored +and normalized compilations separately. Each ordered top-K retains its own +window; the source loader's registered plan covers every window that shares +that source, including subsequent moves. + +Ordered plan aggregation also retains every joined source whose pending demand +gates a retained plan. An outer ordered QueryRef without a join cannot erase +its child's joined dependency. A second ordered join adds its dependency to +the first. Ordered continuation and Collection/Effect publication wait for all +of them. The ordered-work owner checks this composition through actual +Collection and Effect receivers while either joined acquisition is held. + +The runtime owner checks select, where and having source inputs through root, +wrapped child, direct child and real Effect receivers. Predicate-with-select +controls distinguish callback syntax from placement. Collection receivers +observe initial readiness, held mutation, settlement and remote next use. +Effects additionally observe every delivered value and previousValue, +exit/reentry and post-disposal silence. Its bounded evidence is in +[the callback metadata record](../../../../../docs/contributing/oracle-reviews/callback-metadata.md). +Reusing a compilation cache on one graph with the same input streams must retain +the callback inputs and authored selected output that separate compilations +produce. The exported `compileQuery` receiver checks both compilation orders +through FROM, JOIN and union placement at initial graph drain. Its raw output +comparison is differential and supplies no independent Effect-root attribution +policy. Optimizer cloning can prevent an original raw-key cache hit in union +placement; that path checks propagation and transparency instead. +Separate static receiving-type gaps remain in the [deferred oracle record](../../../../../docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md). D2 hashes enumerable symbol keys and uses exact local-symbol identity plus @@ -1532,10 +1562,11 @@ A live query's subscriptions to one collection, such as a join and an include of the same source, share one publication barrier. None of their replays publishes, and none whose last demand retired closes, while another of them still has replay work, so demand that any of them starts meanwhile joins its -own open replay. While a failed replay of any of them holds publication, a +own open replay. While a replay of any of them still holds publication, a lazy demand of any of them that fails after initial-query readiness fails its -own subscription's replacement too, so retiring the other replay's failed -demand cannot publish the window without this demand's rows. It holds until +own subscription's replacement too. This includes a subscription that held no +demand at truncate. Settling a sibling's replay or retiring its failed demand +cannot publish the window without this demand's rows. It holds until every demand load that joins it loads, a later replay of the source succeeds, or the subscription's last demand retires. The query retries the failed demand at once and the retry joins the hold, so the hold ends only after the failed diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index ee60295643..26acc639d8 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -450,12 +450,12 @@ export class CollectionConfigBuilder< const loadOperation = this.liveQueryCollection?._sync.beginLoadSubsetOperation() for (const info of Object.values(this.optimizableOrderByCollections)) { - if (!info.joinedFilterSourceId) continue - for (const plan of this.lazySourcesCallbacks[info.joinedFilterSourceId] - ?.plans ?? []) { - const participant = this.activeDemands.get(plan.id)?.participant - if (participant?.isPending()) { - this.trackSubsetLoadOperationPromise(participant.promise) + for (const sourceId of info.joinedFilterSourceIds ?? []) { + for (const plan of this.lazySourcesCallbacks[sourceId]?.plans ?? []) { + const participant = this.activeDemands.get(plan.id)?.participant + if (participant?.isPending()) { + this.trackSubsetLoadOperationPromise(participant.promise) + } } } } @@ -596,8 +596,8 @@ export class CollectionConfigBuilder< if (this.liveQueryCollection?._lifecycle.status === `ready`) { // The failure settles the demand: the request errors of its keys show // it, and ordered work must not wait for a load that already ended. - // Output held for this demand publishes now, unless a failed replay of - // this source holds publication: the failure then holds this + // Output held for this demand publishes now, unless a replay of this + // source still holds publication: the failure then holds this // subscription's publication too, so retiring the other replay's // failed demand cannot publish without this demand's rows. demand.settled = true @@ -605,7 +605,7 @@ export class CollectionConfigBuilder< this.collectionSources.some( (source) => source.collection === collection && - this.subscriptions[source.sourceId]?.hasFailedTruncateReplacement, + this.subscriptions[source.sourceId]?.hasPendingTruncateReplacement, ) ) subscription.failReplacement(normalized) @@ -1209,10 +1209,10 @@ export class CollectionConfigBuilder< this.orderedLoadFailed || this.hasPendingSourceRecovery() || this.pendingOrderedLoads.size > 0 || - Object.values(this.optimizableOrderByCollections).some( - (info) => - info.joinedFilterSourceId !== undefined && - this.hasPendingJoinedWork(info.joinedFilterSourceId), + Object.values(this.optimizableOrderByCollections).some((info) => + info.joinedFilterSourceIds?.some((sourceId) => + this.hasPendingJoinedWork(sourceId), + ), ) ) { return @@ -1409,8 +1409,8 @@ export class CollectionConfigBuilder< if (status === `ready`) { if ( - Object.values(this.optimizableOrderByCollections).some( - (info) => info.joinedFilterSourceId === sourceId, + Object.values(this.optimizableOrderByCollections).some((info) => + info.joinedFilterSourceIds?.includes(sourceId), ) ) { this.maybeRunGraphFn?.() diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 5fdb3fbe1f..740f9193f7 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -454,10 +454,9 @@ export class CollectionSubscriber< !this.collectionConfigBuilder.hasActiveWindowOperation(), () => this.collectionConfigBuilder.getGraphInputRevision(), () => - orderByInfo.joinedFilterSourceId !== undefined && - this.collectionConfigBuilder.hasPendingJoinedWork( - orderByInfo.joinedFilterSourceId, - ), + orderByInfo.joinedFilterSourceIds?.some((sourceId) => + this.collectionConfigBuilder.hasPendingJoinedWork(sourceId), + ) ?? false, ) this.orderedLoader.start() diff --git a/packages/db/src/query/live/ordered-source-loader.ts b/packages/db/src/query/live/ordered-source-loader.ts index baa705d94b..1b12edf57e 100644 --- a/packages/db/src/query/live/ordered-source-loader.ts +++ b/packages/db/src/query/live/ordered-source-loader.ts @@ -161,7 +161,7 @@ export class OrderedSourceLoader { info.sourceHoldsAllRows && info.index !== undefined && info.orderBy.length === 1 && - info.joinedFilterSourceId === undefined + !info.joinedFilterSourceIds?.length } private get needsOrderingRepair(): boolean { @@ -295,7 +295,10 @@ export class OrderedSourceLoader { this.blockedLoadMore?.windowOperationGeneration continuesOrderedPrefixRepair ||= this.blockedLoadMore?.continuesOrderedPrefixRepair ?? false - if (this.info.joinedFilterSourceId && this.hasPendingJoinedWork()) { + if ( + this.info.joinedFilterSourceIds?.length && + this.hasPendingJoinedWork() + ) { if ( windowOperationGeneration !== undefined || continuesOrderedPrefixRepair @@ -879,7 +882,10 @@ export class OrderedSourceLoader { continuation.continuesOrderedPrefixRepair, ) } else { - if (this.info.joinedFilterSourceId && this.hasPendingJoinedWork()) { + if ( + this.info.joinedFilterSourceIds?.length && + this.hasPendingJoinedWork() + ) { this.stagedContinuation = continuation return } diff --git a/packages/db/tests/query/compiler/basic.test.ts b/packages/db/tests/query/compiler/basic.test.ts index f3209ba94a..53aa58a9d6 100644 --- a/packages/db/tests/query/compiler/basic.test.ts +++ b/packages/db/tests/query/compiler/basic.test.ts @@ -104,8 +104,12 @@ describe(`Query2 Compiler`, () => { const collection = messages[0]! expect(collection.getInner()).toHaveLength(4) - // Check the structure of the results - should be the raw user objects in tuple format - const results = collection.getInner().map(([data]) => data) + // Check the row and ordering slots; later slots carry compiler context. + const results = collection + .getInner() + .map( + ([[key, [row, orderByIndex]]]) => [key, [row, orderByIndex]] as const, + ) expect(results).toContainEqual([1, [sampleUsers[0], undefined]]) expect(results).toContainEqual([2, [sampleUsers[1], undefined]]) expect(results).toContainEqual([3, [sampleUsers[2], undefined]]) @@ -154,8 +158,12 @@ describe(`Query2 Compiler`, () => { graph.run() - // Check the structure of the results - const results = messages[0]!.getInner().map(([data]) => data) + // Check the row and ordering slots; later slots carry compiler context. + const results = messages[0]! + .getInner() + .map( + ([[key, [row, orderByIndex]]]) => [key, [row, orderByIndex]] as const, + ) expect(results).toContainEqual([ 1, diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts index 8a11da8ce3..7ad2fdfd13 100644 --- a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts @@ -165,7 +165,8 @@ import type { LoadSubsetOptions, SyncConfig } from '../../src/types.js' * node may list a failure, and the replay-error identity law skips peer * histories. The replay-error identity law judges only rejections with a failure of a replay * the move waited on; a move that resumes into another failed replay is not - * judged. + * judged. Each subscription owns its replay verdict, even when publication + * is shared with a sibling. Error-order comparisons keep that distinction. */ type OrderedFailureState = `initiating-keyed-request-failed` | `keyed-reacquisition-failed` @@ -316,6 +317,14 @@ type OrderedReach = { // Deferred window moves that rejected with a failure of a replay they // waited on. deferredMoveRejections: number + ownReplayFailureComparisons: number + siblingReplayFailureComparisons: number + siblingSuccessfulCoverage: number + // Owed request-error comparisons after another subscription asked for its keys. + siblingRequestErrorChecks: number + newSiblingReplayFailures: number + newSiblingReplayRetirements: number + newSiblingReplayRecoveries: number replayLoads: number abortRejections: number preloadChecks: number @@ -463,6 +472,11 @@ async function checkOrderedReplay( error?: Error // Its source, the keys it asked for, and whether it is an ordered read. source?: BarrierSource + // Publication is shared, but each subscription owns its replay verdict. + subscription: LoadSubsetOptions[`subscription`] + // This subscription held no acquisition at truncate, then started demand + // during its sibling replay. Measured only from adapter receipts. + newSiblingReplay?: boolean keys?: Set ordered?: boolean } @@ -485,6 +499,8 @@ async function checkOrderedReplay( ambiguous: boolean pending: Map own: Set + // Acquisition receipts held at truncate, used only for declared reach. + subscriptionsAtTruncate: Set } const barriers = new Map() // Every barrier this history raised, in raise order. @@ -514,6 +530,18 @@ async function checkOrderedReplay( ambiguous: [...barriers.values()].some(({ ambiguous }) => ambiguous), pending: new Map(barriers.get(source)?.pending), own: new Set(), + subscriptionsAtTruncate: new Set( + [...acquired].flatMap(([options, count]) => + count > 0 && + loads.some( + (load) => + load.source === source && + load.subscription === options.subscription, + ) + ? [options.subscription] + : [], + ), + ), } barriers.set(source, barrier) raisedBarriers.push(barrier) @@ -540,6 +568,7 @@ async function checkOrderedReplay( barrier.own.add(load) return true } + let replayScopeViolation: string | undefined const settleParticipant = ( source: BarrierSource, load: Load, @@ -561,6 +590,13 @@ async function checkOrderedReplay( barrier.own.has(load) && !options.signal?.aborted ) { + if ( + load.newSiblingReplay && + [...barrier.pending.values()].some( + (pending) => pending.subscription !== options.subscription, + ) + ) + reach.newSiblingReplayFailures++ barrier.failures.set(load, options) barrier.history.push(load) } @@ -569,15 +605,33 @@ async function checkOrderedReplay( // failure no longer fails the replay (ARCHITECTURE.md law 17: a failure // counts for a key no later held load loaded). if (ok && barrier?.pending.has(load) && !options.signal?.aborted) { + // Calibration of the replay account: a successful sibling can supply the + // same source keys without deciding another subscription's verdict. + // Preserve those failure identities independently of the mutation below. + const siblingFailures = [...barrier.failures.keys()].filter( + (failed) => failed.subscription !== load.subscription, + ) for (const failed of barrier.failures.keys()) { if ( + failed.subscription !== load.subscription && loads.indexOf(failed) < loads.indexOf(load) && failed.ordered === load.ordered && failed.keys !== undefined && [...failed.keys].every((key) => load.keys?.has(key)) ) + reach.siblingSuccessfulCoverage++ + if ( + loads.indexOf(failed) < loads.indexOf(load) && + failed.subscription === load.subscription && + failed.ordered === load.ordered && + failed.keys !== undefined && + [...failed.keys].every((key) => load.keys?.has(key)) + ) { barrier.failures.delete(failed) + } } + if (siblingFailures.some((failed) => !barrier.failures.has(failed))) + replayScopeViolation ??= `successful sibling removed another subscription's replay failure` } barrier?.pending.delete(load) // A replay completes when its last participant settles, so demand that @@ -594,7 +648,11 @@ async function checkOrderedReplay( const failures = barrier.failures.size for (const owned of [barrier.pending, barrier.failures]) { for (const [load, held] of owned) { - if (held === options) owned.delete(load) + if (held === options) { + if (owned === barrier.failures && load.newSiblingReplay) + reach.newSiblingReplayRetirements++ + owned.delete(load) + } } } // One failed demand retires while another still holds the barrier. @@ -693,18 +751,19 @@ async function checkOrderedReplay( // request error on the query's own node. The error stays until one of these // meets it: // - its demand retires, which aborts its request and unloads it; - // - a later load of the source asks for some of its keys: that newer + // - a later load of the same subscription asks for some of its keys: that newer // request of the demand decides them, and keys it no longer asks for // left the demand (ARCHITECTURE.md law 16: a lazy demand's error clears // when that demand settles or is released; a key's latest load decides // it); - // - a later load of the same source covers its keys; + // - a later load of the same subscription covers its keys; // - a replay of its source that rose after the failure completes, since a // replay asks again for every held demand. // A failure that joins a replay waits for its subscription's verdict, // which comes once that subscription's participants settled, so it owes // no error before then. The barrier that holds publication stays shared - // across the query's subscriptions to the source. Keys come from evaluating the request's + // across the query's subscriptions to the source. A sibling's successful + // load cannot decide this demand's error. Keys come from evaluating the request's // `where` over a small key domain, independent of production's bookkeeping. // The account combines the production concepts of a failed request and the // replay verdict that decides it; it names no production structure. @@ -734,6 +793,7 @@ async function checkOrderedReplay( const requestFailures: Array = [] const loadedKeys: Array<{ source: BarrierSource + subscription: LoadSubsetOptions[`subscription`] keys: Set at: number }> = [] @@ -758,7 +818,25 @@ async function checkOrderedReplay( ) ) reach.successWhileSubscriptionOwes++ - loadedKeys.push({ source, keys, at: ++requestClock }) + // Later successful coverage by this subscription is a recovery control. + // Coverage by a sibling cannot decide this subscription's failure. + if ( + loads.some( + (earlier) => + earlier.newSiblingReplay && + earlier.outcome === `failed` && + earlier.source === source && + earlier.subscription === options.subscription && + [...(earlier.keys ?? [])].every((key) => keys.has(key)), + ) + ) + reach.newSiblingReplayRecoveries++ + loadedKeys.push({ + source, + subscription: options.subscription, + keys, + at: ++requestClock, + }) } const recordRequestFailure = ( load: Load, @@ -799,6 +877,7 @@ async function checkOrderedReplay( (later, index) => index > at && later.source === failure.source && + later.subscription === failure.options.subscription && !later.ordered && [...failure.keys].some((key) => later.keys?.has(key)), ) @@ -806,6 +885,7 @@ async function checkOrderedReplay( const covered = loadedKeys.some( (loaded) => loaded.source === failure.source && + loaded.subscription === failure.options.subscription && loaded.at > failure.at && [...failure.keys].every((key) => loaded.keys.has(key)), ) @@ -890,6 +970,13 @@ async function checkOrderedReplay( requestErrorChecks: 0, successWhileSubscriptionOwes: 0, deferredMoveRejections: 0, + ownReplayFailureComparisons: 0, + siblingReplayFailureComparisons: 0, + siblingSuccessfulCoverage: 0, + siblingRequestErrorChecks: 0, + newSiblingReplayFailures: 0, + newSiblingReplayRetirements: 0, + newSiblingReplayRecoveries: 0, replayLoads: 0, abortRejections: 0, preloadChecks: 0, @@ -942,11 +1029,20 @@ async function checkOrderedReplay( // owner, whose release unloads it. acquired.set(options, (acquired.get(options) ?? 0) + 1) const load: Load = { + subscription: options.subscription, epoch: truncates, outcome: `failed`, source: `markers`, keys: requestedKeys(`markers`, options), } + load.newSiblingReplay = + barriers.get(`markers`) !== undefined && + !barriers + .get(`markers`)! + .subscriptionsAtTruncate.has(options.subscription) && + [...(barriers.get(`markers`)?.pending.values() ?? [])].some( + (pending) => pending.subscription !== options.subscription, + ) loads.push(load) reach.markerLoads++ // The throw happens while the subscription registers the @@ -990,11 +1086,20 @@ async function checkOrderedReplay( const startedReady = allReady() acquired.set(options, (acquired.get(options) ?? 0) + 1) const load: Load = { + subscription: options.subscription, epoch: truncates, outcome: `pending`, source: `markers`, keys: requestedKeys(`markers`, options), } + load.newSiblingReplay = + barriers.get(`markers`) !== undefined && + !barriers + .get(`markers`)! + .subscriptionsAtTruncate.has(options.subscription) && + [...(barriers.get(`markers`)?.pending.values() ?? [])].some( + (pending) => pending.subscription !== options.subscription, + ) loads.push(load) reach.markerLoads++ const participant = joinBarrier(`markers`, load, options) @@ -1146,6 +1251,7 @@ async function checkOrderedReplay( const startedReady = allReady() acquired.set(options, (acquired.get(options) ?? 0) + 1) const load: Load = { + subscription: options.subscription, epoch: truncates, outcome: `pending`, source: `rows`, @@ -1348,6 +1454,19 @@ async function checkOrderedReplay( if (owed.length === 0) return reach.requestErrorChecks++ for (const failure of owed) { + // Reach is measured at the health assertion, after a sibling asked + // for overlapping keys that cannot decide this subscription's error. + if ( + loads.some( + (later, index) => + index > loads.indexOf(failure.load) && + later.source === failure.source && + later.subscription !== failure.options.subscription && + !later.ordered && + [...failure.keys].some((key) => later.keys?.has(key)), + ) + ) + reach.siblingRequestErrorChecks++ expect( [live, peer].some((query) => query?.health.error?.errors.some( @@ -1367,7 +1486,12 @@ async function checkOrderedReplay( // failing, as a demand's replacement does, no longer count. A failure of a // load in no waited replay is the move's own window failure after it // resumed, which this law does not judge. + // The source-wide barrier combines subscriptions only for publication. + // Their verdicts remain separate: a sibling can fail while another replay + // still loads. Only failures of the rejecting subscription compete for its + // earliest error. Subscription identity comes from adapter request data. const covers = (later: Load, earlier: Load) => + later.subscription === earlier.subscription && later.ordered === earlier.ordered && later.keys !== undefined && earlier.keys !== undefined && @@ -1392,13 +1516,32 @@ async function checkOrderedReplay( ) if (candidates.length === 0) return reach.deferredMoveRejections++ + const rejected = loads.find((load) => load.error === error)! + for (const barrier of candidates) { + if ( + barrier.history.filter( + (load) => load.subscription === rejected.subscription, + ).length > 1 + ) + reach.ownReplayFailureComparisons++ + if ( + barrier.history.some( + (load) => load.subscription !== rejected.subscription, + ) + ) + reach.siblingReplayFailureComparisons++ + } const earliest = candidates.some((barrier) => barrier.history .slice( 0, barrier.history.findIndex((load) => load.error === error), ) - .every((earlier, at) => !stillDecides(barrier, earlier, at)), + .every( + (earlier, at) => + earlier.subscription !== rejected.subscription || + !stillDecides(barrier, earlier, at), + ), ) if (!earliest) moveErrorViolation ??= `move ${move} rejected with "${error.message}", not its replay's earliest failure that still decides it` @@ -1441,9 +1584,10 @@ async function checkOrderedReplay( const raised = [...barriers.values()] if (raised.some(({ ambiguous }) => ambiguous)) return if (live.status === `error`) return - expect(JSON.parse(JSON.stringify(live.toArray)), label).toEqual( - JSON.parse(retainedRows), - ) + expect( + JSON.parse(JSON.stringify(live.toArray)), + `${label}; new sibling replay failures: ${reach.newSiblingReplayFailures}`, + ).toEqual(JSON.parse(retainedRows)) if (raised.some(({ failures }) => failures.size > 0)) { reach.retainedFailedReplays++ } else reach.heldPublicationChecks++ @@ -1697,6 +1841,8 @@ async function checkOrderedReplay( if (moveOrderViolation) { throw new Error(`newest move wins: ${moveOrderViolation}`) } + if (replayScopeViolation) + throw new Error(`replay account scope: ${replayScopeViolation}`) if (moveErrorViolation) { throw new Error(`replay error identity: ${moveErrorViolation}`) } @@ -2429,6 +2575,194 @@ describe(`Ordered windows across truncate replays`, () => { ) }) + // A window with no initial include demand can activate its first include + // acquisition while a sibling join replay is still loading. Failure of that + // demand belongs to the shared publication barrier (law 9), even though its + // subscription had no replay of its own at the truncate. Adapter receipts + // prove both premises. This bounded grammar varies order, width, prior + // settlements and later demand changes; it reuses every ordinary observation + // and the independent retention account above. A move can retire the failed + // demand; a later truncate can replace it with successful own coverage. + it(`keeps new include failures behind a loading sibling replay`, async () => { + const states = [ + `newSiblingReplayFailures`, + `newSiblingReplayRetirements`, + `newSiblingReplayRecoveries`, + ] as const + const declared = createOracleReach(`replay.new-sibling-demand`, states) + for (const direction of [`desc`, `asc`] as const) { + for (const limit of [2, 1, 3] as const) { + for (const settles of [2, 1]) { + for (const change of [`mark`, `none`, `move`, `replay`] as const) { + declared.beginHistory() + const observed = await checkOrderedReplay( + [ + { type: `hold` }, + { type: `truncateMarkers` }, + ...Array.from({ length: settles }, () => ({ + type: `settle` as const, + ok: true, + keepHolding: true, + newest: false, + })), + { type: `settle`, ok: false, newest: true, keepHolding: true }, + ...(change === `none` + ? [] + : change === `replay` + ? [{ type: `truncateMarkers` as const }] + : change === `mark` + ? [{ type: `mark` as const, id: 1, on: false }] + : [ + { + type: `move` as const, + id: 1, + rank: direction === `desc` ? 60 : 1, + }, + ]), + ], + { + ...ordinaryOrderedScenario, + direction, + offset: 1, + limit, + joined: `none`, + includes: true, + markerMode: `on-demand`, + }, + ) + for (const state of states) { + if (observed[state] > 0) declared.reach(state) + } + } + } + } + } + console.info( + `replay.new-sibling-demand`, + Object.fromEntries(declared.counts()), + ) + declared.assertReached() + }, 120_000) + + // This finite history grammar varies which source replays, the order of + // two load failures, join form, direction and window width. Join plus include + // gives sibling subscriptions; the root replay gives same-subscription + // failures. The same comparison checks both, without an expected error + // copied from production or a source-wide error order. + it(`keeps replay error order within each subscription`, async () => { + const declared = createOracleReach( + `replay.subscription-error-order`, + [ + `ownReplayFailureComparisons`, + `siblingReplayFailureComparisons`, + ] as const, + { minimum: 5 }, + ) + for (const direction of [`asc`, `desc`] as const) { + for (const joined of [false, `some`, `none`, `inner`] as const) { + for (const limit of [1, 2, 3] as const) { + for (const markers of [false, true]) { + for (const newest of [false, true]) { + declared.beginHistory() + const observed = await checkOrderedReplay( + [ + { type: `hold` }, + markers + ? { type: `truncateMarkers` } + : { type: `truncate`, armAfterStart: false }, + { type: `window`, offset: 0, limit }, + { type: `settle`, ok: false, newest }, + { + type: `settle`, + ok: false, + newest: !newest, + keepHolding: false, + }, + ], + { + ...ordinaryOrderedScenario, + direction, + joined, + limit, + markerMode: `on-demand`, + includes: true, + }, + ) + for (const state of [ + `ownReplayFailureComparisons`, + `siblingReplayFailureComparisons`, + ] as const) { + if (observed[state] > 0) declared.reach(state) + } + } + } + } + } + } + console.info( + `replay.subscription-error-order`, + Object.fromEntries(declared.counts()), + ) + declared.assertReached() + }, 120_000) + + // Failure followed by successful coverage is the neighboring regime of + // the two-failure grammar above. Real adapter subscriptions supply identity; + // the shared account comparison requires a sibling's failure to survive. + it(`retains sibling replay failures through successful coverage`, async () => { + const declared = createOracleReach( + `replay.subscription-success-scope`, + [`siblingSuccessfulCoverage`, `siblingRequestErrorChecks`] as const, + { minimum: 1 }, + ) + for (const direction of [`asc`, `desc`] as const) { + for (const joined of [false, `some`, `none`, `inner`] as const) { + for (const limit of [1, 2, 3] as const) { + for (const markers of [false, true]) { + for (const newest of [false, true]) { + declared.beginHistory() + const observed = await checkOrderedReplay( + [ + { type: `hold` }, + markers + ? { type: `truncateMarkers` } + : { type: `truncate`, armAfterStart: false }, + { type: `window`, offset: 0, limit }, + { type: `settle`, ok: false, newest, keepHolding: true }, + { + type: `settle`, + ok: true, + newest: !newest, + keepHolding: false, + }, + ], + { + ...ordinaryOrderedScenario, + direction, + joined, + limit, + markerMode: `on-demand`, + includes: true, + }, + ) + for (const state of [ + `siblingSuccessfulCoverage`, + `siblingRequestErrorChecks`, + ] as const) { + if (observed[state] > 0) declared.reach(state) + } + } + } + } + } + } + console.info( + `replay.subscription-success-scope`, + Object.fromEntries(declared.counts()), + ) + declared.assertReached() + }, 120_000) + it.each(orderedReplayCampaigns)( `keeps the ordered window under load and release faults (%s campaign)`, async (campaign) => { @@ -2455,6 +2789,13 @@ describe(`Ordered windows across truncate replays`, () => { requestErrorChecks: 0, successWhileSubscriptionOwes: 0, deferredMoveRejections: 0, + ownReplayFailureComparisons: 0, + siblingReplayFailureComparisons: 0, + siblingSuccessfulCoverage: 0, + siblingRequestErrorChecks: 0, + newSiblingReplayFailures: 0, + newSiblingReplayRetirements: 0, + newSiblingReplayRecoveries: 0, replayLoads: 0, abortRejections: 0, preloadChecks: 0, @@ -2493,6 +2834,14 @@ describe(`Ordered windows across truncate replays`, () => { `discardedReplayMoves`, `requestErrorChecks`, `deferredMoveRejections`, + `ownReplayFailureComparisons`, + `siblingReplayFailureComparisons`, + `siblingSuccessfulCoverage`, + `siblingRequestErrorChecks`, + // The constructive new-sibling grammar declares these rare premises. + `newSiblingReplayFailures`, + `newSiblingReplayRetirements`, + `newSiblingReplayRecoveries`, `successWhileSubscriptionOwes`, ]) // Counts are histories, rather than repeated events in one trace. diff --git a/packages/db/tests/query/ordered-compiler-cache-driver.ts b/packages/db/tests/query/ordered-compiler-cache-driver.ts new file mode 100644 index 0000000000..f5ef947cda --- /dev/null +++ b/packages/db/tests/query/ordered-compiler-cache-driver.ts @@ -0,0 +1,238 @@ +/** Controlled compiler/ordered-loader receiver for the ordered-work owner. + * One real on-demand Collection supplies a finite provider prefix. Two roots + * consume the exact authored QueryIR on one D2 graph, with shared or separate + * caches. Captures retain signed multiplicities; the owner supplies row truth. + * This is a core compiler/loader cut, not a public Collection window promise. + */ +import { D2, MultiSet, output } from '@tanstack/db-ivm' +import { createCollection } from '../../src/collection/index.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { Query, compileQuery, eq } from '../../src/query/index.js' +import { getQueryIR } from '../../src/query/builder/index.js' +import { QueryRef, UnionAll } from '../../src/query/ir.js' +import { OrderedSourceLoader } from '../../src/query/live/ordered-source-loader.js' +import { withOracleCleanup } from '../utils.js' +import type { CollectionSubscription } from '../../src/collection/subscription.js' +import type { OrderByOptimizationInfo } from '../../src/query/compiler/order-by.js' +import type { + QueryCache, + WindowOptions, +} from '../../src/query/compiler/types.js' +import type { QueryIR } from '../../src/query/ir.js' +import type { + LoadSubsetOptions, + LoadSubsetRequestResult, + SyncConfig, +} from '../../src/types.js' + +type Row = { id: number; rank: number } +export type OrderedCompilerCacheHistory = { + order: `authored-first` | `callback-first` + cache: `shared` | `separate` + path: `query-ref` | `joined-source` | `union-branch` + shape: `structured` | `functional` + direction: `asc` | `desc` + initial: { limit: number; offset: number } + moved: { limit: number; offset: number } +} +type Request = LoadSubsetOptions & { + onLoadSubsetResult?: ( + result: LoadSubsetRequestResult, + options: LoadSubsetOptions, + release: () => void, + ) => void +} +export async function receiveOrderedCompilerCacheHistory( + history: OrderedCompilerCacheHistory, + rows: ReadonlyArray, + check: ( + cut: `initial` | `moved` | `returned`, + actual: { authored: Array; callback: Array }, + ) => void, +) { + // The provider order is part of this controlled adapter's premise. This + // sorts provider data, not the expected answer; the owner computes that. + const provider = [...rows].sort((a, b) => + history.direction === `asc` ? a.rank - b.rank : b.rank - a.rank, + ) + let sync!: Parameters[`sync`]>[0] + const source = createCollection({ + id: `ordered-compiler-cache`, + getKey: (row: Row) => row.id, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: (params) => { + sync = params + sync.begin() + sync.write({ type: `insert`, value: { ...provider[0]! } }) + sync.commit() + sync.markReady() + return { loadSubset: () => true } + }, + }, + }) + let loader: OrderedSourceLoader | undefined + await withOracleCleanup(async () => { + const base = new Query() + .from({ chosen: source }) + .orderBy(({ chosen }) => chosen.rank, history.direction) + .offset(history.initial.offset) + .limit(history.initial.limit) + const selected = getQueryIR( + history.shape === `structured` + ? base.select(({ chosen }) => ({ + id: chosen.id, + rank: chosen.rank, + $key: `spoof`, + })) + : base.fn.select(({ chosen }) => ({ + id: chosen.id, + rank: chosen.rank, + $key: `spoof`, + })), + ) + const joined = getQueryIR( + new Query() + .from({ peer: source }) + .leftJoin( + { nested: new Query().from({ chosen: source }) }, + ({ peer, nested }) => eq(peer.id, nested.id), + ), + ) + joined.join![0]!.from = new QueryRef( + selected, + `nested`, + joined.join![0]!.from.bindingId, + ) + const consumed: QueryIR = + history.path === `joined-source` + ? { ...joined, fnSelect: ({ nested }) => nested } + : history.path === `union-branch` + ? { from: new UnionAll([selected]), fnSelect: (row) => row } + : { + from: new QueryRef(selected, `nested`), + fnSelect: ({ nested }) => nested, + } + const graph = new D2() + const input = graph.newInput<[number, Row]>() + const cache: QueryCache = new WeakMap() + const plans: Record = {} + let move!: (options: WindowOptions) => void + const compile = (query: QueryIR) => + compileQuery( + query, + { chosen: input, peer: input }, + { [source.id]: source }, + {}, + {}, + new Set(), + plans, + query === selected + ? (fn) => { + move = fn + } + : () => {}, + history.cache === `shared` ? cache : new WeakMap(), + ) + const [authored, callback] = + history.order === `authored-first` + ? [compile(selected), compile(consumed)] + : (() => { + const other = compile(consumed) + return [compile(selected), other] + })() + const authoredBag = new Map() + const callbackBag = new Map() + for (const [compiled, bag] of [ + [authored, authoredBag], + [callback, callbackBag], + ] as const) + compiled.pipeline.pipe( + output((message) => { + for (const [[, [value]], weight] of message.getInner()) { + if (value === undefined) continue // legal unmatched LEFT-join side + const id = (value as Row).id + bag.set(id, (bag.get(id) ?? 0) + weight) + } + }), + ) + graph.finalize() + input.sendData( + new MultiSet(source.toArray.map((row) => [[row.id, row as Row], 1])), + ) + graph.run() + let revision = 0 + const request = (options: Request, boundary: boolean) => { + if (!boundary) { + const page = provider.slice( + options.offset ?? 0, + (options.offset ?? 0) + (options.limit ?? provider.length), + ) + const fresh = page.filter((row) => !source.has(row.id)) + if (fresh.length) { + sync.begin() + for (const row of fresh) + sync.write({ type: `insert`, value: { ...row } }) + sync.commit() + revision++ + input.sendData( + new MultiSet( + fresh.map((row) => [[row.id, source.get(row.id)! as Row], 1]), + ), + ) + graph.run() + } + } + options.onLoadSubsetResult?.(true, options, () => {}) + } + const subscription = { + setOrderByIndex: () => {}, + readOrderedSnapshot: (options: LoadSubsetOptions) => + source.toArray + .sort((a, b) => + history.direction === `asc` ? a.rank - b.rank : b.rank - a.rank, + ) + .slice(0, options.limit ?? source.size) + .map((value) => ({ key: value.id, value })), + requestLimitedSnapshot: (options: Request) => request(options, false), + // All ranks are unique. A tie request adds no row beyond the installed + // boundary; it still settles its real loader continuation. + requestSnapshot: (options: Request) => request(options, true), + } + const info = Object.values(plans)[0]! + loader = new OrderedSourceLoader( + info, + subscription as unknown as CollectionSubscription, + `chosen`, + () => {}, + () => false, + () => revision, + ) + const snapshot = () => { + const expand = (bag: Map) => + [...bag].flatMap(([id, count]) => { + if (!Number.isSafeInteger(count) || count < 0) + throw new Error(`invalid ordered multiplicity ${id}:${count}`) + return Array.from({ length: count }, () => id) + }) + return { authored: expand(authoredBag), callback: expand(callbackBag) } + } + loader.start() + await loader.loadMore(0) + graph.run() + check(`initial`, snapshot()) + move(history.moved) + graph.run() + await loader.loadMore(1) + graph.run() + check(`moved`, snapshot()) + move(history.initial) + graph.run() + await loader.loadMore(2) + graph.run() + check(`returned`, snapshot()) + }, [() => loader?.dispose(), () => source.cleanup()]) +} diff --git a/packages/db/tests/query/ordered-joined-plan-driver.ts b/packages/db/tests/query/ordered-joined-plan-driver.ts new file mode 100644 index 0000000000..af92ae19a7 --- /dev/null +++ b/packages/db/tests/query/ordered-joined-plan-driver.ts @@ -0,0 +1,239 @@ +/** Real Collection and Effect receivers for the ordered-work owner. Adapter + * holds control joined acquisition; observations include every publication and + * the current public result at each cut. The owner supplies independent truth. + */ +import { createCollection } from '../../src/collection/index.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { + Query, + createLiveQueryCollection, + eq, + isUndefined, + not, +} from '../../src/query/index.js' +import { createEffect } from '../../src/query/effect.js' +import { evaluateReferenceExpression } from '../reference-expression-oracle.js' +import { flushPromises, withOracleCleanup } from '../utils.js' +import type { LoadSubsetOptions, SyncConfig } from '../../src/types.js' + +export type OrderedJoinedPlanHistory = { + receiver: `collection` | `effect` + direction: `asc` | `desc` + filter: `some` | `none` + placement: `direct` | `wrapped` | `two-filters` + width: 1 | 2 + releaseFirst: `first` | `second` + blocker: `first` | `second` +} +export type OrderedPlanRow = { id: number; rank: number } +export type OrderedJoinedPlanObservation = { + cut: `initial` | `held` | `partial` | `settled` + pending: number + rows: Array + publications: Array> +} +export async function receiveOrderedJoinedPlanHistory( + history: OrderedJoinedPlanHistory, + check: (observation: OrderedJoinedPlanObservation) => void, +) { + type Marker = { id: number; rowId: number } + let rowSync!: Parameters[`sync`]>[0] + let holding = false + const held: Array<{ + source: `first` | `second` + options: LoadSubsetOptions + answer: () => void + }> = [] + const rows = createCollection({ + id: `ordered-plan-rows`, + getKey: (row: OrderedPlanRow) => row.id, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + syncMode: `on-demand`, + sync: { + sync: (operations) => { + rowSync = operations + operations.begin() + for (const id of [1, 2, 3]) + operations.write({ type: `insert`, value: { id, rank: id } }) + operations.commit() + operations.markReady() + return { loadSubset: () => true } + }, + }, + }) + const markerSource = (source: `first` | `second`, marked: Array) => { + const collection = createCollection({ + id: `ordered-plan-${source}`, + getKey: (marker: Marker) => marker.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (operations) => { + operations.markReady() + return { + loadSubset: (options) => { + const answer = () => { + if (options.signal?.aborted) return + operations.begin() + for (const id of marked) { + const value = { id, rowId: id } + if ( + options.where && + evaluateReferenceExpression(options.where, value) !== true + ) + continue + operations.write({ + type: collection.has(id) ? `update` : `insert`, + value, + }) + } + operations.commit() + } + if (holding) + return new Promise((resolve) => + held.push({ + source, + options, + answer: () => { + answer() + resolve() + }, + }), + ) + answer() + return true + }, + } + }, + }, + }) + return collection + } + const firstSource = markerSource( + `first`, + history.blocker === `first` ? [1, 4] : [1], + ) + const secondSource = markerSource( + `second`, + history.blocker === `second` ? [1, 4] : [1], + ) + const inner = new Query() + .from({ row: rows }) + .leftJoin({ first: firstSource }, ({ row, first }) => + eq(row.id, first.rowId), + ) + .where(({ first }) => + history.filter === `none` + ? isUndefined(first.id) + : not(isUndefined(first.id)), + ) + .orderBy(({ row }) => row.rank, history.direction) + .limit(history.placement === `direct` ? history.width : 3) + .select(({ row }) => row) + const query = + history.placement === `direct` + ? inner + : history.placement === `wrapped` + ? new Query() + .from({ nested: inner }) + .orderBy(({ nested }) => nested.rank, history.direction) + .limit(history.width) + .select(({ nested }) => nested) + : new Query() + .from({ nested: inner }) + .leftJoin({ second: secondSource }, ({ nested, second }) => + eq(nested.id, second.rowId), + ) + .where(({ second }) => + history.filter === `none` + ? isUndefined(second.id) + : not(isUndefined(second.id)), + ) + .orderBy(({ nested }) => nested.rank, history.direction) + .limit(history.width) + .select(({ nested }) => nested) + const effectRows = new Map() + let publications: Array> = [] + const readEffect = () => + [...effectRows.values()] + .sort((a, b) => + history.direction === `asc` ? a.rank - b.rank : b.rank - a.rank, + ) + .map(({ id }) => id) + let cleanupReceiver: () => void | Promise = () => {} + let read = readEffect + await withOracleCleanup(async () => { + if (history.receiver === `collection`) { + const live = createLiveQueryCollection(() => query) + read = () => live.toArray.map(({ id }) => id) + const subscription = live.subscribeChanges(() => + publications.push(read()), + ) + cleanupReceiver = async () => { + subscription.unsubscribe() + await live.cleanup() + } + await live.preload() + } else { + const effect = createEffect({ + query: () => query, + onBatch: (events) => { + for (const event of events) { + if (event.type === `exit`) effectRows.delete(event.key) + else effectRows.set(event.key, { ...event.value }) + } + publications.push(readEffect()) + }, + }) + cleanupReceiver = () => effect.dispose() + } + await flushPromises() + const cut = (name: OrderedJoinedPlanObservation[`cut`]) => { + check({ + cut: name, + rows: read(), + pending: held.filter(({ options }) => !options.signal?.aborted).length, + publications, + }) + publications = [] + } + const answer = async (source?: `first` | `second`) => { + for (let turn = 0; turn < 50; turn++) { + const at = held.findIndex( + (load) => source === undefined || load.source === source, + ) + if (at < 0) return + held.splice(at, 1)[0]!.answer() + await flushPromises() + } + throw new Error(`joined plan receiver did not drain`) + } + cut(`initial`) + holding = true + rowSync.begin() + rowSync.write({ + type: `insert`, + value: { id: 4, rank: history.direction === `asc` ? 0 : 4 }, + }) + rowSync.commit() + await flushPromises() + cut(`held`) + await answer(history.releaseFirst) + cut(`partial`) + holding = false + await answer() + await flushPromises() + cut(`settled`) + }, [ + async () => { + holding = false + for (const load of held.splice(0)) load.answer() + await cleanupReceiver() + }, + () => rows.cleanup(), + () => firstSource.cleanup(), + () => secondSource.cleanup(), + ]) +} diff --git a/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts b/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts index df1ceebb1e..4f8f4ef9b1 100644 --- a/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts +++ b/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts @@ -516,7 +516,7 @@ async function checkAuthorityCase( // A synchronous repair answer reads installed rows: an eager source. ...(testCase.syncRepair ? { sourceHoldsAllRows: true } : {}), requiresFullSource: testCase.requiresFullSource, - ...(testCase.joined ? { joinedFilterSourceId: `markers` } : {}), + ...(testCase.joined ? { joinedFilterSourceIds: [`markers`] } : {}), // A window that needs more always needs one row, as behind a joined // filter that drops rows. Otherwise it needs the window's rows that do // not sort at or before the loaded boundary. diff --git a/packages/db/tests/query/ordered-source-loader-state.test.ts b/packages/db/tests/query/ordered-source-loader-state.test.ts index a633d793cb..4e080489a6 100644 --- a/packages/db/tests/query/ordered-source-loader-state.test.ts +++ b/packages/db/tests/query/ordered-source-loader-state.test.ts @@ -93,7 +93,7 @@ describe(`Ordered source request ownership`, () => { let joinedPending = false const loader = new OrderedSourceLoader( createOrderByInfo({ - joinedFilterSourceId: `child`, + joinedFilterSourceIds: [`child`], requiresFullSource: true, }), fakeSubscription(requests, releases), diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index fd11aed199..6b283bd586 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -1,5 +1,5 @@ import { fc, test as fcTest } from '@fast-check/vitest' -import { describe, expect, it, vi } from 'vitest' +import { afterAll, describe, expect, it, vi } from 'vitest' import { toArray } from '../../src/query/index.js' import { createCollection } from '../../src/collection/index.js' import { localOnlyCollectionOptions } from '../../src/local-only.js' @@ -20,6 +20,8 @@ import { evaluateReferenceExpression } from '../reference-expression-oracle.js' import { createOracleReach } from '../oracle-reach.js' import { flushPromises } from '../utils.js' import { withHistoryCleanup } from '../optimistic-history-oracle.js' +import { receiveOrderedCompilerCacheHistory } from './ordered-compiler-cache-driver.js' +import { receiveOrderedJoinedPlanHistory } from './ordered-joined-plan-driver.js' import { compareRows, expectDistinctRequests, @@ -7911,3 +7913,232 @@ declareOracle({ ], grammar: [exhaustiveScenarios, eagerFullJoinCase], }) + +/** A source's ordered acquisition must serve every compiled window that reads + * that lexical source. Authored/source metadata modes may have separate result + * streams; overwriting one window's demand with the other's cannot change the + * independent sorted slice. Authority: ordered acquisition and window laws in + * ARCHITECTURE.md. The exported compiler plus real loader is a controlled core + * receiving cut; the existing live Collection/Effect histories supply the host + * gate. These finite histories do not promise a universal operator-count bound. + */ +const compilerCacheWindows = ( + [`authored-first`, `callback-first`] as const +).flatMap((order) => + ([`shared`, `separate`] as const).flatMap((cache) => + ([`query-ref`, `joined-source`, `union-branch`] as const).flatMap((path) => + ([`structured`, `functional`] as const).flatMap((shape) => + ([`asc`, `desc`] as const).flatMap((direction) => + [ + { + initial: { limit: 1, offset: 0 }, + moved: { limit: 3, offset: 0 }, + }, + { + initial: { limit: 3, offset: 0 }, + moved: { limit: 1, offset: 0 }, + }, + { + initial: { limit: 1, offset: 0 }, + moved: { limit: 2, offset: 1 }, + }, + ].map((window) => ({ + order, + cache, + path, + shape, + direction, + ...window, + })), + ), + ), + ), + ), +) +const compilerWindowCuts = [`initial`, `moved`, `returned`] as const +const compilerWindowKey = (history: (typeof compilerCacheWindows)[number]) => + JSON.stringify(history) +const compilerWindowReach = createOracleReach( + `compiled ordered cache windows`, + compilerCacheWindows.flatMap((history) => + compilerWindowCuts.map((cut) => `${compilerWindowKey(history)}/${cut}`), + ), + { replaying: false }, +) +let compilerWindowEntered = false +afterAll(() => { + if (compilerWindowEntered) compilerWindowReach.assertReached() +}) +declareOracle({ + model: [expectedInstalledWindow], + grammar: [compilerCacheWindows], +}) +it.each(compilerCacheWindows)( + `serves every metadata-mode window of one ordered source: %j`, + async (history) => { + compilerWindowEntered = true + compilerWindowReach.beginHistory() + const rows = [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + { id: 3, rank: 3 }, + ] + const failures: Array = [] + const traces: Array = [] + await receiveOrderedCompilerCacheHistory(history, rows, (cut, actual) => { + const window = cut === `moved` ? history.moved : history.initial + const expected = { + authored: expectedInstalledWindow(rows, { + ...window, + direction: history.direction, + }), + callback: expectedInstalledWindow(rows, { + ...history.initial, + direction: history.direction, + }), + } + // Compare signed-bag occurrences, not D2 output delivery order. Rank order + // is independently exercised by the enclosing ordered-work owner. + const byId = (ids: Array) => [...ids].sort((a, b) => a - b) + try { + expect( + { authored: byId(actual.authored), callback: byId(actual.callback) }, + `${compilerWindowKey(history)}/${cut}`, + ).toStrictEqual({ + authored: byId(expected.authored), + callback: byId(expected.callback), + }) + } catch (error) { + failures.push(error) + traces.push({ cut, actual, expected }) + } + compilerWindowReach.reach(`${compilerWindowKey(history)}/${cut}`) + }) + if (failures.length) + throw new AggregateError( + failures, + `ordered compiler cache window mismatch: ${JSON.stringify(traces)}`, + { cause: failures[0] }, + ) + }, +) + +/** Ordered publication waits for every joined dependency that any retained + * source plan requires (ARCHITECTURE.md, atomic window publication). A QueryRef + * adds ordering without removing its child's lazy join. Another ordered join + * adds a dependency rather than replacing it. The model filters complete source + * rows, then sorts and slices. While required adapter loads are held, every + * public read and callback retains the preceding complete result. + * + * This finite history grammar varies receiving host, join polarity, nested + * plan composition, window width, direction and which joined source answers + * first. Direct plans are controls. It uses actual Collection/Effect receivers + * with controlled adapters, not a provider or framework timing claim. + */ +const joinedPlanHistories = ([`collection`, `effect`] as const).flatMap( + (receiver) => + ([`asc`, `desc`] as const).flatMap((direction) => + ([`some`, `none`] as const).flatMap((filter) => + ([`direct`, `wrapped`, `two-filters`] as const).flatMap((placement) => + ([1, 2] as const).flatMap((width) => + ([`first`, `second`] as const).flatMap((releaseFirst) => + (placement === `two-filters` + ? ([`first`, `second`] as const) + : ([`first`] as const) + ).map((blocker) => ({ + receiver, + direction, + filter, + placement, + width, + releaseFirst, + blocker, + })), + ), + ), + ), + ), + ), +) +function expectedJoinedPlanWindow( + history: (typeof joinedPlanHistories)[number], + added: boolean, +) { + const rows = [1, 2, 3].map((id) => ({ id, rank: id })) + if (added) rows.push({ id: 4, rank: history.direction === `asc` ? 0 : 4 }) + const markers = + history.placement === `two-filters` + ? [ + history.blocker === `first` ? [1, 4] : [1], + history.blocker === `second` ? [1, 4] : [1], + ] + : [[1, 4]] + const accepted = rows.filter(({ id }) => + markers.every((marked) => + history.filter === `some` ? marked.includes(id) : !marked.includes(id), + ), + ) + return expectedInstalledWindow(accepted, { + direction: history.direction, + offset: 0, + limit: history.width, + }) +} +declareOracle({ + model: [expectedJoinedPlanWindow], + grammar: [joinedPlanHistories], +}) +const joinedPlanReach = createOracleReach( + `ordered joined plan publication`, + joinedPlanHistories.flatMap((history) => [ + `${JSON.stringify(history)}/held`, + `${JSON.stringify(history)}/settled`, + ]), + { replaying: false }, +) +let joinedPlanEntered = false +afterAll(() => { + if (joinedPlanEntered) joinedPlanReach.assertReached() +}) +it.each(joinedPlanHistories)( + `retains every joined dependency through ordered plans: %j`, + async (history) => { + joinedPlanEntered = true + joinedPlanReach.beginHistory() + const failures: Array = [] + const traces: Array = [] + await receiveOrderedJoinedPlanHistory(history, (observation) => { + const added = observation.cut !== `initial` && observation.pending === 0 + const expected = expectedJoinedPlanWindow(history, added) + try { + expect( + observation.rows, + `${JSON.stringify(history)}/${observation.cut}`, + ).toEqual(expected) + // Ordinary Effect startup may deliver progressive callbacks + // (ARCHITECTURE.md, atomic window publication). This history starts its + // publication claim after readiness; the initial settled rows establish + // the preceding complete result. + if (observation.cut !== `initial`) + for (const publication of observation.publications) + expect( + publication, + `complete ${observation.cut} publication`, + ).toEqual(expected) + } catch (error) { + failures.push(error) + traces.push({ ...observation, expected }) + } + if (observation.cut === `held` && observation.pending > 0) + joinedPlanReach.reach(`${JSON.stringify(history)}/held`) + if (observation.cut === `settled` && observation.pending === 0) + joinedPlanReach.reach(`${JSON.stringify(history)}/settled`) + }) + if (failures.length) + throw new AggregateError( + failures, + `ordered joined plan publication mismatch: ${JSON.stringify(traces)}`, + { cause: failures[0] }, + ) + }, +) diff --git a/packages/db/tests/query/ordered-work.test.ts b/packages/db/tests/query/ordered-work.test.ts index a8353e33cf..a55c61846d 100644 --- a/packages/db/tests/query/ordered-work.test.ts +++ b/packages/db/tests/query/ordered-work.test.ts @@ -207,7 +207,7 @@ describe(`ordered source work oracle`, () => { const builder = getCollectionBuilder(live)! expect( Object.values(builder.optimizableOrderByCollections).some( - (info) => info.joinedFilterSourceId !== undefined, + (info) => (info.joinedFilterSourceIds?.length ?? 0) > 0, ), ).toBe(true) const scheduled = vi.spyOn(builder, `scheduleGraphRun`) diff --git a/packages/db/tests/query/virtual-row-fields-input-driver.ts b/packages/db/tests/query/virtual-row-fields-input-driver.ts new file mode 100644 index 0000000000..6f15a64e81 --- /dev/null +++ b/packages/db/tests/query/virtual-row-fields-input-driver.ts @@ -0,0 +1,1264 @@ +/** + * Production receiver for the callback-input oracle. Callbacks copy their facts + * immediately, including missing versus own undefined. At an action, D2 may + * invoke a callback for old-row retractions as well as new-row contributions; + * the check permits those two independently authored states. Startup LEFT joins + * may expose an absent joined side. Established output cuts require both rows. + * No callback count, ordering, or fully matched startup snapshot is promised. + */ +import { inspect, isDeepStrictEqual } from 'node:util' +import { D2, MultiSet, output } from '@tanstack/db-ivm' +import { createCollection } from '../../src/collection/index.js' +import { + Query, + compileQuery, + createLiveQueryCollection, + eq, + include, + materialize, + toArray, +} from '../../src/query/index.js' +import { createEffect } from '../../src/query/effect.js' +import { getQueryIR } from '../../src/query/builder/index.js' +import { PropRef, QueryRef, UnionAll } from '../../src/query/ir.js' +import { + flushPromises, + mockSyncCollectionOptions, + withOracleCleanup, +} from '../utils.js' +import { virtualFieldNames } from './virtual-row-fields-support.js' +import { + authoredMetadataCollision, + expectedMetadataInput, + expectedMetadataRecord, + metadataInputViolations, + metadataOutputViolations, +} from './virtual-row-fields-input-oracle.js' +import type { Collection } from '../../src/collection/index.js' +import type { QueryBuilder } from '../../src/query/builder/index.js' +import type { RefLeaf } from '../../src/query/builder/types.js' +import type { QueryCache } from '../../src/query/compiler/types.js' +import type { QueryIR } from '../../src/query/ir.js' +import type { + MetadataCallbackKind, + MetadataCallbackPath, + MetadataCompilerCachePath, + MetadataInputDescription, + MetadataInputRow, + MetadataInputSnapshot, + MetadataSourceFact, +} from './virtual-row-fields-input-oracle.js' + +type Output = { id: string | number; observed: MetadataInputSnapshot } +type OutputContext = { + schema: {} + baseSchema: {} + fromSourceName: string + result: Output + hasResult: true +} +type ProjectionContext = { + schema: {} + baseSchema: {} + fromSourceName: string + result: MetadataInputRow + hasResult: true +} +export type CallbackInputCut = + | `initial` + | `held` + | `settled` + | `remote-next-use` + | `exit` + | `reentry` + | `disposed` +export type CallbackInputReport = { + violations: Array<{ + cut: CallbackInputCut + kind: string + actual: unknown + expected: unknown + }> + callbackCounts: Record +} + +// Throw only after every cut. Keeping the full report on the primary failure +// lets withOracleCleanup preserve it if resource release also fails. +class MetadataCallbackOracleFailure extends Error { + constructor(readonly report: CallbackInputReport) { + super( + `Callback-input law: ${inspect(report.violations[0], { depth: null })}`, + ) + this.name = `MetadataCallbackOracleFailure` + } +} + +function copyData(value: unknown): unknown { + if (Array.isArray(value)) { + const copy = value.map(copyData) + // Array.map retains indices and holes but omits named own fields. Keep + // those fields too, so capture cannot erase a wrong inline array shape. + for (const key of Object.getOwnPropertyNames(value)) { + if (Object.hasOwn(copy, key)) continue + Object.defineProperty(copy, key, { + value: copyData(Reflect.get(value, key)), + enumerable: true, + configurable: true, + writable: true, + }) + } + return copy + } + if (value && typeof value === `object`) + return Object.fromEntries( + Object.getOwnPropertyNames(value).map((key) => [ + key, + copyData((value as Record)[key]), + ]), + ) + return value +} +// This recorder reads the actual public callback value. Its expected counterpart +// is authored from source facts and projection syntax in the oracle module. +// Top-level reserved fields are read through their public properties; nested +// inline records retain exact selected own-field shape through copyData. +export function captureMetadataInput(value: unknown): MetadataInputSnapshot { + const record = + value !== null && typeof value === `object` + ? (value as Record) + : undefined + return { + present: record !== undefined, + ordinary: Object.fromEntries( + Object.getOwnPropertyNames(record ?? {}) + .filter((key) => !virtualFieldNames.some((field) => field === key)) + .map((key) => [key, copyData(record![key])]), + ), + fields: Object.fromEntries( + virtualFieldNames.map((field) => [ + field, + { + present: record !== undefined && field in record, + own: record !== undefined && Object.hasOwn(record, field), + value: copyData(record?.[field]), + }, + ]), + ), + } +} + +function projectInput( + collection: Collection, + alias: string, + description: MetadataInputDescription, + parentGroup?: RefLeaf, +) { + const all = new Query().from({ [alias]: collection }) + const base = + parentGroup === undefined + ? all + : all.where((refs) => eq(refs[alias]!.parentId, parentGroup)) + const overrides = authoredMetadataCollision(description) + const shape = description.shape + const selected = + shape === `functional-direct` + ? base.fn.select((refs) => refs[alias]!) + : shape === `functional-spread` + ? base.fn.select((refs) => ({ ...refs[alias]!, ...overrides })) + : shape === `functional-projection` + ? base.fn.select((refs) => ({ + id: refs[alias]!.id, + label: refs[alias]!.label, + optional: undefined, + ...overrides, + })) + : shape === `structured-spread` + ? base.select((refs) => ({ ...refs[alias]!, ...overrides })) + : base.select((refs) => ({ + id: refs[alias]!.id, + label: refs[alias]!.label, + hasPendingWrites: refs[alias]!.$hasPendingWrites, + ...overrides, + })) + // Context erasure changes only TS's union of authoring forms. The production + // query and exact runtime fields remain checked, without casting observations. + return selected as unknown as QueryBuilder +} +function freshRows(): Array { + return [ + { id: 1, parentId: `group`, label: `number-before`, optional: undefined }, + { id: `1`, parentId: `group`, label: `string-before`, optional: undefined }, + ] +} +function source(name: string, rows: ReadonlyArray) { + return createCollection( + mockSyncCollectionOptions({ + id: name, + getKey: (row) => row.id, + initialData: rows.map((row) => ({ ...row })), + }), + ) +} +function outputRows(rows: ReadonlyArray): Array { + return rows.map( + (row) => + Object.fromEntries( + Object.entries(row).filter( + ([key]) => !virtualFieldNames.some((field) => field === key), + ), + ) as Output, + ) +} +function rowsFromSnapshots( + snapshots: ReadonlyArray, +): Array { + return snapshots.map((snapshot) => ({ + ...snapshot.ordinary, + ...Object.fromEntries( + Object.entries(snapshot.fields) + .filter(([, field]) => field.present) + .map(([name, field]) => [name, field.value]), + ), + })) +} + +/** Every history uses actual source writes and a real held mutation handler. + * Comparisons accumulate so a known initial RED cannot hide later reach. */ +export async function runMetadataCallbackHistory( + description: MetadataInputDescription, + path: MetadataCallbackPath | `direct-source`, + reached: (cut: CallbackInputCut) => void, + receiver: `root` | `inline-child` | `inline-child-direct` | `effect` = `root`, + callbackKind: MetadataCallbackKind = `select`, +): Promise { + const initial = freshRows() + const union = path.startsWith(`union`) + const first = source(`callback-first`, union ? [initial[0]!] : initial) + const second = source(`callback-second`, [initial[1]!]) + const peers = source(`callback-peer`, initial) + const containingParents = source(`callback-containing-parent`, initial) + let facts: Array = initial.map((row, index) => ({ + row: { ...row }, + sourceId: union && index === 1 ? second.id : first.id, + pending: false, + })) + let cut: CallbackInputCut = `initial` + let allowed = facts.map((fact) => expectedMetadataInput(fact, description)) + let enteredCallback = `select` + const captures: Array<{ + cut: CallbackInputCut + callback: string + input: MetadataInputSnapshot + }> = [] + const project = (row: unknown, id: string | number) => { + const observed = captureMetadataInput(row) + // A flat union HAVING callback also receives the compiler's selected-value + // context. It is not an ordinary field of the source value being judged. + // Capture reserved fields before removing that known context from ordinary + // fields; spreading first could hide inherited or nonenumerable metadata. + if (union && enteredCallback === `having`) + delete observed.ordinary.$selected + captures.push({ cut, callback: enteredCallback, input: observed }) + return { id, observed } + } + const makeSelectQuery = (parentId?: RefLeaf) => { + const input = projectInput(first, `chosen`, description) + if (path === `direct-source`) { + const direct = new Query().from({ direct: first }) + return ( + parentId === undefined + ? direct + : direct.where(({ direct: row }) => eq(row.id, parentId)) + ).fn.select(({ direct: row }) => project(row, row.id)) + } + if (union) { + const other = projectInput(second, `other`, description) + const combined = + path === `union-last` + ? new Query().unionAll(other, input) + : new Query().unionAll(input, other) + if (path === `union-query-ref`) { + const outer = new Query().from({ nested: combined }) + return ( + parentId === undefined + ? outer + : outer.where(({ nested }) => eq(nested.id, parentId)) + ).fn.select(({ nested }) => project(nested, nested.id)) + } + return ( + parentId === undefined + ? combined + : combined.where((row) => eq(row.id, parentId)) + ).fn.select((row) => project(row, row.id)) + } + if (path === `joined-source`) { + const joined = new Query() + .from({ peer: peers }) + .leftJoin({ nested: input }, ({ peer, nested }) => + eq(peer.id, nested.id), + ) + return ( + parentId === undefined + ? joined + : joined.where(({ peer }) => eq(peer.id, parentId)) + ).fn.select(({ peer, nested }) => project(nested, peer.id)) + } + if (path === `joined-from`) { + const joined = new Query() + .from({ nested: input }) + .leftJoin({ peer: peers }, ({ nested, peer }) => eq(nested.id, peer.id)) + return ( + parentId === undefined + ? joined + : joined.where(({ nested }) => eq(nested.id, parentId)) + ).fn.select(({ nested }) => project(nested, nested.id)) + } + const once = new Query().from({ nested: input }) + if (path === `nested-query-ref`) { + const outer = new Query().from({ outer: once }) + return ( + parentId === undefined + ? outer + : outer.where(({ outer: row }) => eq(row.id, parentId)) + ).fn.select(({ outer: row }) => project(row, row.id)) + } + return ( + parentId === undefined + ? once + : once.where(({ nested }) => eq(nested.id, parentId)) + ).fn.select(({ nested }) => project(nested, nested.id)) + } + const selects = callbackKind === `select` || callbackKind.endsWith(`-select`) + const makeQuery = (parentId?: RefLeaf) => { + const authored = makeSelectQuery(parentId) + if (callbackKind === `select`) return authored + const ir = getQueryIR(authored) + const recordSelection = ir.fnSelect! + const predicateKind = callbackKind.startsWith(`where`) ? `where` : `having` + const predicate = (row: Parameters[0]) => { + enteredCallback = predicateKind + try { + const derived = recordSelection(row) as Output + // A real predicate derives membership from the observed reserved field. + // It uses no expected answer or production provenance classification. + return derived.observed.fields.$key?.value !== `spoof-$key` + } finally { + enteredCallback = `select` + } + } + if (predicateKind === `where`) + ir.fnWhere = [...(ir.fnWhere ?? []), predicate] + else ir.fnHaving = [...(ir.fnHaving ?? []), predicate] + if (!selects) { + delete ir.fnSelect + if (!union) { + const alias = + path === `direct-source` + ? `direct` + : path === `nested-query-ref` + ? `outer` + : `nested` + ir.select = { + id: new PropRef([path === `joined-source` ? `peer` : alias, `id`]), + value: new PropRef([alias]), + } + } + } + // The builder receives its authored IR back; this changes callback syntax, + // not the receiver's production path or expected source facts. + return authored + } + const observeRows = (rows: ReadonlyArray): Array => + selects + ? outputRows(rows) + : rows.map((row) => { + const record = row as Record + const observed = captureMetadataInput(union ? row : record.value) + return { + id: record.id as string | number, + observed: receiver.startsWith(`inline-child`) + ? observed + : expectedMetadataRecord( + observed.present ? observed.ordinary : undefined, + ), + } + }) + const expectedOutput = (fact: MetadataSourceFact) => + selects + ? expectedMetadataInput(fact, description) + : receiver.startsWith(`inline-child`) + ? expectedMetadataInput(fact, description, true) + : expectedMetadataRecord( + expectedMetadataInput(fact, description, true).ordinary, + ) + const query = + receiver === `root` || receiver === `effect` + ? makeQuery() + : new Query() + .from({ parent: containingParents }) + .select(({ parent }) => ({ + id: parent.id, + // The direct receiver avoids an extra QueryRef that could sanitize + // the entire child recursively and conceal a missed union boundary. + children: + receiver === `inline-child-direct` + ? toArray( + makeQuery( + parent.id, + ) as unknown as QueryBuilder, + ) + : toArray( + new Query() + .from({ + result: + makeQuery() as unknown as QueryBuilder, + }) + .where(({ result }) => eq(result.id, parent.id)), + ), + })) + // A real Effect receives the same model and callback grammar through both + // public channels. Its ledgers retain each value/previousValue before another + // action. No implicit Effect-root metadata policy is inferred here. + const effectRows = new Map() + const namedRows = new Map() + const effectErrors: Array = [] + const effectEvents: Array<{ + type: string + key: string | number + value: unknown + previousValue?: unknown + }> = [] + const namedEvents: typeof effectEvents = [] + const receive = ( + rows: typeof effectRows, + events: typeof effectEvents, + event: { + type: string + key: string | number + value: unknown + previousValue?: unknown + }, + ) => { + const captured = copyData(event) as (typeof effectEvents)[number] + events.push(captured) + const previous = rows.get(event.key) + if ( + event.type === `update` && + !isDeepStrictEqual(previous, event.previousValue) + ) + effectErrors.push({ + kind: `Effect previousValue differs from prior delivered value`, + previous, + event: captured, + }) + if (event.type === `enter` && previous !== undefined) + effectErrors.push({ + kind: `Effect enters an active occurrence`, + event: captured, + }) + if (event.type !== `enter` && previous === undefined) + effectErrors.push({ + kind: `Effect changes an absent occurrence`, + event: captured, + }) + if (event.type === `exit`) rows.delete(event.key) + else rows.set(event.key, copyData(event.value) as Output) + } + const effect = + receiver === `effect` + ? createEffect({ + query: () => query as unknown as QueryBuilder, + onBatch: (events) => { + for (const event of events) receive(effectRows, effectEvents, event) + }, + onEnter: (event) => receive(namedRows, namedEvents, event), + onUpdate: (event) => receive(namedRows, namedEvents, event), + onExit: (event) => receive(namedRows, namedEvents, event), + onError: (error) => { + effectErrors.push(error) + }, + onSourceError: (error) => { + effectErrors.push(error) + }, + }) + : undefined + const live = effect + ? undefined + : createLiveQueryCollection({ + query: () => query as unknown as QueryBuilder, + getKey: (row: Output) => row.id, + }) + const report: CallbackInputReport = { + violations: [], + callbackCounts: { + initial: 0, + held: 0, + settled: 0, + 'remote-next-use': 0, + exit: 0, + reentry: 0, + disposed: 0, + }, + } + let checkedEffectEvents = 0 + const check = async () => { + if (effect) await flushPromises() + const premise = + (live?.isReady() ?? (!effect?.disposed && first.isReady())) && + first.get(1)?.$hasPendingWrites === (cut === `held`) + if (!premise) + report.violations.push({ + cut, + kind: `readiness or held-source premise missing`, + actual: { + ready: live?.isReady() ?? (!effect?.disposed && first.isReady()), + pending: first.get(1)?.$hasPendingWrites, + }, + expected: { ready: true, pending: cut === `held` }, + }) + const current = captures.filter((capture) => capture.cut === cut) + report.callbackCounts[cut] = current.length + const requiredCallback = callbackKind.startsWith(`where`) + ? `where` + : callbackKind.startsWith(`having`) + ? `having` + : `select` + const callbacksEntered = + current.some((capture) => capture.callback === requiredCallback) && + (!selects || current.some((capture) => capture.callback === `select`)) + if (!callbacksEntered) + report.violations.push({ + cut, + kind: `callback not entered`, + actual: current, + expected: `one or more`, + }) + for (const capture of current) { + const permitted = + path === `joined-source` + ? [...allowed, expectedMetadataRecord(undefined)] + : allowed + for (const kind of metadataInputViolations(capture.input, permitted)) + report.violations.push({ + cut, + kind, + actual: capture.input, + expected: permitted, + }) + } + const expected = facts.map((fact) => ({ + id: fact.row.id, + observed: expectedOutput(fact), + })) + // The unmatched LEFT-join peer remains a result after its source exits. + // The model's absent snapshot is judged at callbacks and output separately. + if (path === `joined-source` && cut === `exit`) + expected.push({ id: `1`, observed: expectedMetadataRecord(undefined) }) + const raw = + receiver === `effect` + ? [...effectRows.values()] + : receiver === `root` + ? live!.toArray + : live!.toArray.flatMap( + (row) => (row as unknown as { children: Array }).children, + ) + if (effect) { + // Check each delivered value, not only the final active rows. Retraction + // evaluation can legally use before/after facts; it cannot deliver a + // snapshot outside those independently authored source states. + const permitted = allowed.flatMap((observed) => [ + { + id: observed.ordinary.id, + observed: selects + ? observed + : expectedMetadataRecord( + observed.present ? observed.ordinary : undefined, + ), + }, + ]) + if (path === `joined-source`) + for (const row of initial) + permitted.push({ + id: row.id, + observed: expectedMetadataRecord(undefined), + }) + for (const event of effectEvents.slice(checkedEffectEvents)) { + for (const value of [ + event.value, + ...(event.type === `update` ? [event.previousValue] : []), + ]) { + const actual = observeRows([value as object]) + if ( + !permitted.some( + (row) => + metadataOutputViolations(actual, [row as Output]).length === 0, + ) + ) + report.violations.push({ + cut, + kind: `Effect delivered value outside authored states`, + actual, + expected: permitted, + }) + } + } + checkedEffectEvents = effectEvents.length + } + if (effect) { + if (effectErrors.length || !isDeepStrictEqual(effectEvents, namedEvents)) + report.violations.push({ + cut, + kind: `Effect callback channels or event continuity`, + actual: { effectErrors, effectEvents, namedEvents }, + expected: `matching complete channels and continuous values`, + }) + for (const kind of metadataOutputViolations( + observeRows([...namedRows.values()]), + expected, + )) + report.violations.push({ + cut, + kind: `named Effect: ${kind}`, + actual: [...namedRows.values()], + expected, + }) + } + const actual = observeRows(raw) + for (const kind of metadataOutputViolations(actual, expected)) + report.violations.push({ cut, kind, actual, expected }) + // Reach records the actual callback path and established source/output + // boundary even when the law comparison is RED. It is not a passing count. + if (premise && callbacksEntered) reached(cut) + } + let finish = async () => {} + await withOracleCleanup(async () => { + if (live) await live.preload() + await check() + cut = `held` + const before = facts.map((fact) => expectedMetadataInput(fact, description)) + facts = facts.map((fact) => + fact.row.id === 1 + ? { + ...fact, + row: { ...fact.row, label: `number-pending` }, + pending: true, + } + : fact, + ) + allowed = [ + ...before, + ...facts.map((fact) => expectedMetadataInput(fact, description)), + ] + const transaction = first.update(1, (row) => { + row.label = `number-pending` + }) + finish = async () => { + first.utils.resolveSync() + await transaction.when(`settled`) + finish = async () => {} + } + await check() + cut = `settled` + const pending = facts.map((fact) => + expectedMetadataInput(fact, description), + ) + facts = facts.map((fact) => + fact.row.id === 1 + ? { ...fact, row: { ...initial[0]! }, pending: false } + : fact, + ) + allowed = [ + ...pending, + ...facts.map((fact) => expectedMetadataInput(fact, description)), + ] + await finish() + await check() + cut = `remote-next-use` + const settled = facts.map((fact) => + expectedMetadataInput(fact, description), + ) + facts = facts.map((fact) => + fact.row.id === `1` + ? { ...fact, row: { ...fact.row, label: `string-remote` } } + : fact, + ) + allowed = [ + ...settled, + ...facts.map((fact) => expectedMetadataInput(fact, description)), + ] + const receivingSource = union ? second : first + receivingSource.utils.begin() + receivingSource.utils.write({ type: `update`, value: { ...facts[1]!.row } }) + receivingSource.utils.commit() + await check() + if (effect) { + // Source exit/reentry exercises onExit and a fresh active interval. The + // same before/after authored facts judge both channels at each cut. + cut = `exit` + const departing = facts[1]! + allowed = facts.map((fact) => expectedMetadataInput(fact, description)) + facts = [facts[0]!] + const receiving = union ? second : first + // Use the actual branch owning the string key; no model source is moved. + receiving.utils.begin() + receiving.utils.write({ type: `delete`, value: { ...departing.row } }) + receiving.utils.commit() + await check() + cut = `reentry` + facts = [ + ...facts, + { ...departing, row: { ...departing.row, label: `string-reentry` } }, + ] + allowed = facts.map((fact) => expectedMetadataInput(fact, description)) + receiving.utils.begin() + receiving.utils.write({ type: `insert`, value: { ...facts[1]!.row } }) + receiving.utils.commit() + await check() + cut = `disposed` + await effect.dispose() + const beforeDisposal = { + captures: captures.length, + batch: effectEvents.length, + named: namedEvents.length, + } + receiving.utils.begin() + receiving.utils.write({ + type: `update`, + value: { ...facts[1]!.row, label: `after-disposal` }, + }) + receiving.utils.commit() + await flushPromises() + const afterDisposal = { + captures: captures.length, + batch: effectEvents.length, + named: namedEvents.length, + } + if (!effect.disposed || !isDeepStrictEqual(beforeDisposal, afterDisposal)) + report.violations.push({ + cut, + kind: `Effect delivery after disposal`, + actual: afterDisposal, + expected: beforeDisposal, + }) + else reached(cut) + } + if (report.violations.length) + throw new MetadataCallbackOracleFailure(report) + }, [ + () => finish(), + () => effect?.dispose(), + () => live?.cleanup(), + () => containingParents.cleanup(), + () => peers.cleanup(), + () => second.cleanup(), + () => first.cleanup(), + ]) + return report +} + +/** The inline receiver exercises the real materializer with authored child + * values. The same source/input judgment predicts them; no second metadata + * model is hidden in a receiving test. Compiled Collections reject at admission. */ +export async function runInlineMetadataCallbackHistory( + description: MetadataInputDescription, + form: `array` | `materialized` | `collection`, + reached: (cut: CallbackInputCut | `rejected`) => void, + path: `direct` | `query-ref` = `direct`, +): Promise { + const initial = freshRows() + const children = source(`callback-inline-child`, initial) + const parentRows: Array = + path === `direct` + ? [ + { + id: `parent`, + parentId: `group`, + label: `parent`, + optional: undefined, + }, + ] + : initial.map((row) => ({ ...row })) + const parents = source(`callback-inline-parent`, parentRows) + let facts = initial.map((row) => ({ + row: { ...row }, + sourceId: children.id, + pending: false, + })) + let cut: CallbackInputCut = `initial` + // A QueryRef in a child without a functional callback preserves the inline + // selected shape too. One parent per typed key gives its outer correlation + // a named source field without requiring a projected parentId. + const expected = () => + parentRows.map((parent) => + expectedMetadataRecord({ + id: parent.id, + children: rowsFromSnapshots( + facts + .filter((fact) => path === `direct` || fact.row.id === parent.id) + .map((fact) => expectedMetadataInput(fact, description, true)), + ), + }), + ) + let allowed = expected() + // The controlled source publishes its initial child batch atomically. A + // startup callback can precede that batch or consume its complete rows. + allowed.push( + ...parentRows.map((parent) => + expectedMetadataRecord({ id: parent.id, children: [] }), + ), + ) + const calls: Array<{ cut: CallbackInputCut; input: MetadataInputSnapshot }> = + [] + const build = () => + createLiveQueryCollection({ + query: (q) => { + const included = q.from({ parent: parents }).select(({ parent }) => { + const selected = + path === `direct` + ? projectInput( + children, + `child`, + description, + parent.parentId, + ).orderBy(({ $selected }) => $selected.label) + : new Query() + .from({ + nested: projectInput(children, `child`, description), + }) + .where(({ nested }) => eq(nested.id, parent.id)) + .orderBy(({ nested }) => nested.label) + return { + id: parent.id, + children: + form === `collection` + ? include( + selected as unknown as QueryBuilder, + ) + : form === `array` + ? toArray( + selected as unknown as QueryBuilder, + ) + : materialize( + selected as unknown as QueryBuilder, + ), + } + }) + return q.from({ nested: included }).fn.select(({ nested }) => { + const observed = captureMetadataInput(nested) + calls.push({ cut, input: observed }) + return { id: nested.id, observed } + }) + }, + getKey: (row: Output) => row.id, + }) + const report: CallbackInputReport = { + violations: [], + callbackCounts: { + initial: 0, + held: 0, + settled: 0, + 'remote-next-use': 0, + exit: 0, + reentry: 0, + disposed: 0, + }, + } + if (form === `collection`) { + let unexpected: ReturnType | undefined + await withOracleCleanup(() => { + let error: unknown + try { + unexpected = build() + } catch (caught) { + error = caught + } + if ( + !(error instanceof Error) || + !error.message.includes( + `fn.select() cannot consume Collection-valued includes`, + ) || + calls.length > 0 + ) + report.violations.push({ + cut, + kind: `compiled Collection input admitted or callback entered`, + actual: { error, calls }, + expected: `rejection before callback`, + }) + else reached(`rejected`) + if (report.violations.length) + throw new MetadataCallbackOracleFailure(report) + }, [ + () => unexpected?.cleanup(), + () => parents.cleanup(), + () => children.cleanup(), + ]) + return report + } + const live = build() + const check = () => { + const premise = + live.isReady() && children.get(1)?.$hasPendingWrites === (cut === `held`) + if (!premise) + report.violations.push({ + cut, + kind: `inline readiness or held-source premise missing`, + actual: { + ready: live.isReady(), + pending: children.get(1)?.$hasPendingWrites, + }, + expected: { ready: true, pending: cut === `held` }, + }) + const current = calls.filter((call) => call.cut === cut) + report.callbackCounts[cut] = current.length + if (!current.length) + report.violations.push({ + cut, + kind: `inline callback not entered`, + actual: current, + expected: `one or more`, + }) + for (const call of current) + for (const kind of metadataInputViolations(call.input, allowed)) + report.violations.push({ + cut, + kind, + actual: call.input, + expected: allowed, + }) + const actual = outputRows(live.toArray) + const next = expected().map((observed, index) => ({ + id: parentRows[index]!.id, + observed, + })) + for (const kind of metadataOutputViolations(actual, next)) + report.violations.push({ cut, kind, actual, expected: next }) + if (premise && current.length && live.toArray.length === parentRows.length) + reached(cut) + } + let finish = async () => {} + await withOracleCleanup(async () => { + await live.preload() + check() + cut = `held` + const before = expected() + facts = facts.map((fact) => + fact.row.id === 1 + ? { + ...fact, + row: { ...fact.row, label: `number-pending` }, + pending: true, + } + : fact, + ) + allowed = [...before, ...expected()] + const transaction = children.update(1, (row) => { + row.label = `number-pending` + }) + finish = async () => { + children.utils.resolveSync() + await transaction.when(`settled`) + finish = async () => {} + } + check() + cut = `settled` + const pending = expected() + facts = facts.map((fact) => + fact.row.id === 1 + ? { ...fact, row: { ...initial[0]! }, pending: false } + : fact, + ) + allowed = [...pending, ...expected()] + await finish() + check() + cut = `remote-next-use` + const settled = expected() + facts = facts.map((fact) => + fact.row.id === `1` + ? { ...fact, row: { ...fact.row, label: `string-remote` } } + : fact, + ) + allowed = [...settled, ...expected()] + children.utils.begin() + children.utils.write({ type: `update`, value: { ...facts[1]!.row } }) + children.utils.commit() + check() + if (report.violations.length) + throw new MetadataCallbackOracleFailure(report) + }, [ + () => finish(), + () => live.cleanup(), + () => parents.cleanup(), + () => children.cleanup(), + ]) + return report +} + +/** The low-level compiler receives the exact same selected QueryIR twice on + * one D2 graph. Unlike builder placement, this public entry point need not + * clone that query. Each history crosses compilation order and cache sharing. + * + * Only the initial graph-drain cut is claimed here. Public source Collection + * rows supply genuine metadata; independent authored facts predict callback + * input and derived output. Raw selected output is compared across shared and + * separate caches; no implicit metadata law is assigned to an Effect root. */ +export async function runMetadataCompilerCacheHistory( + description: MetadataInputDescription, + order: `authored-first` | `callback-first`, + cache: `shared` | `separate`, + path: MetadataCompilerCachePath, + reached: ( + observation: `callback` | `derived-output` | `authored-output`, + ) => void, +): Promise { + const initial = freshRows() + const first = source(`callback-compiler-cache`, initial) + const facts: Array = initial.map((row) => ({ + row: { ...row }, + sourceId: first.id, + pending: false, + })) + const report: CallbackInputReport = { + violations: [], + callbackCounts: { + initial: 0, + held: 0, + settled: 0, + 'remote-next-use': 0, + exit: 0, + reentry: 0, + disposed: 0, + }, + } + await withOracleCleanup(async () => { + await first.preload() + const execute = (sharing: `shared` | `separate`) => { + const selected = getQueryIR(projectInput(first, `chosen`, description)) + const captures: Array = [] + const capture = (value: unknown, id?: string | number) => { + const observed = captureMetadataInput(value) + captures.push(observed) + return { id: id ?? (value as MetadataInputRow).id, observed } + } + const joined = getQueryIR( + new Query() + .from({ peer: first }) + .leftJoin( + { nested: projectInput(first, `chosen`, description) }, + ({ peer, nested }) => eq(peer.id, nested.id), + ), + ) + // Install the exact shared IR after authoring the legal join expression. + // Query builders normally clone placements, which concealed the cache bug. + joined.join![0]!.from = new QueryRef( + selected, + `nested`, + joined.join![0]!.from.bindingId, + ) + const consumed: QueryIR = + path === `joined-source` + ? { + ...joined, + fnSelect: ({ nested, peer }) => + capture(nested, (peer as MetadataInputRow).id), + } + : path === `union-branch` + ? { + from: new UnionAll([selected]), + fnSelect: (value) => capture(value), + } + : { + from: new QueryRef(selected, `nested`), + fnSelect: ({ nested }) => capture(nested), + } + const graph = new D2() + const input = graph.newInput<[string | number, MetadataInputRow]>() + const sharedCache: QueryCache = new WeakMap() + const readCache = sharedCache.get.bind(sharedCache) + let exactQueryReads = 0 + sharedCache.get = (query) => { + if (query === selected) exactQueryReads++ + return readCache(query) + } + const compile = (query: QueryIR) => + compileQuery( + query, + { chosen: input, peer: input }, + { [first.id]: first }, + {}, + {}, + new Set(), + {}, + () => {}, + sharing === `shared` ? sharedCache : new WeakMap(), + ) + const [authored, callback] = + order === `authored-first` + ? [compile(selected), compile(consumed)] + : (() => { + const compiledCallback = compile(consumed) + return [compile(selected), compiledCallback] + })() + const authoredDeltas: Array<{ row: Output; weight: number }> = [] + const derivedDeltas: Array<{ row: Output; weight: number }> = [] + authored.pipeline.pipe( + output((message) => { + for (const [[, [value]], weight] of message.getInner()) + authoredDeltas.push({ + row: { + id: (value as MetadataInputRow).id, + observed: captureMetadataInput(value), + }, + weight, + }) + }), + ) + callback.pipeline.pipe( + output((message) => { + for (const [[, [value]], weight] of message.getInner()) { + const row = value as Output + derivedDeltas.push({ + row: { + id: row.id, + // Copy the selected ordinary capture at output delivery too. + observed: copyData(row.observed) as MetadataInputSnapshot, + }, + weight, + }) + } + }), + ) + graph.finalize() + input.sendData( + new MultiSet(first.toArray.map((row) => [[row.id, row], 1])), + ) + graph.run() + + return { captures, authoredDeltas, derivedDeltas, exactQueryReads } + } + const { captures, authoredDeltas, derivedDeltas, exactQueryReads } = + execute(cache) + const reference = execute(`separate`) + if ( + process.env.TANSTACK_DB_ORACLE_STATISTICS === `1` && + description.shape === `structured-projection` && + description.collision === `none` && + cache === `shared` + ) + console.info(`compiler exact raw-query reads`, { + path, + order, + reads: exactQueryReads, + }) + + // Independent signed-bag bookkeeping preserves duplicate contributions and + // retractions. It does not replace the source/projection judgment. + const accumulate = ( + deltas: ReadonlyArray<{ row: Output; weight: number }>, + ) => { + const counts: Array<{ row: Output; weight: number }> = [] + for (const delta of deltas) { + const existing = counts.find((entry) => + isDeepStrictEqual(entry.row, delta.row), + ) + if (existing) existing.weight += delta.weight + else counts.push({ ...delta }) + } + const rows: Array = [] + for (const entry of counts) { + if (!Number.isSafeInteger(entry.weight) || entry.weight < 0) { + report.violations.push({ + cut: `initial`, + kind: `invalid compiled output multiplicity`, + actual: entry, + expected: `a nonnegative integer at graph drain`, + }) + continue + } + for (let occurrence = 0; occurrence < entry.weight; occurrence++) + rows.push(entry.row) + } + return rows + } + const actualAuthored = accumulate(authoredDeltas) + const actualDerived = accumulate(derivedDeltas) + const expectedInput = facts.map((fact) => + expectedMetadataInput(fact, description), + ) + const expectedDerived = facts.map((fact, index) => ({ + id: fact.row.id, + observed: expectedInput[index]!, + })) + // Cache transparency is a relation, not an Effect-root attribution rule. + // Separate compilation supplies the other execution, never the O12 truth. + // The independent source/projection model still judges callback inputs and + // complete ordinary derived outputs in both executions below. + const expectedAuthored = accumulate(reference.authoredDeltas) + const referenceDerived = accumulate(reference.derivedDeltas) + for (const capture of reference.captures) + for (const kind of metadataInputViolations( + capture, + path === `joined-source` + ? [...expectedInput, expectedMetadataRecord(undefined)] + : expectedInput, + )) + report.violations.push({ + cut: `initial`, + kind: `reference: ${kind}`, + actual: capture, + expected: expectedInput, + }) + for (const kind of metadataOutputViolations( + referenceDerived, + expectedDerived, + )) + report.violations.push({ + cut: `initial`, + kind: `reference: ${kind}`, + actual: referenceDerived, + expected: expectedDerived, + }) + report.callbackCounts.initial = captures.length + if (!captures.length) + report.violations.push({ + cut: `initial`, + kind: `compiler callback not entered`, + actual: captures, + expected: `one or more`, + }) + for (const capture of captures) + for (const kind of metadataInputViolations( + capture, + path === `joined-source` + ? [...expectedInput, expectedMetadataRecord(undefined)] + : expectedInput, + )) + report.violations.push({ + cut: `initial`, + kind, + actual: capture, + expected: expectedInput, + }) + for (const [name, actual, expected] of [ + [`derived`, actualDerived, expectedDerived], + [`authored`, actualAuthored, expectedAuthored], + ] as const) + for (const kind of metadataOutputViolations(actual, expected)) + report.violations.push({ + cut: `initial`, + kind: `${name}: ${kind}`, + actual, + expected, + }) + // Reach records executed comparisons, including RED comparisons, after the + // real source and both compiled roots reach the named graph-drain cut. + if (first.isReady() && captures.length) reached(`callback`) + if (first.isReady() && actualDerived.length === facts.length) + reached(`derived-output`) + if (first.isReady() && actualAuthored.length === facts.length) + reached(`authored-output`) + if (report.violations.length) + throw new MetadataCallbackOracleFailure(report) + }, [() => first.cleanup()]) + return report +} diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts new file mode 100644 index 0000000000..06358ec70e --- /dev/null +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts @@ -0,0 +1,469 @@ +/** + * # Reserved fields must be correct before user JavaScript consumes them. + * + * The shared input oracle states the maintainer's source/projection/inline law. + * This finite receiver exhausts 1,155 root histories, 1,155 wrapped child + * histories, 825 direct child histories and 1,155 real Effect histories. + * Each crosses select, where, having and predicate-with-select controls. + * Another 198 inline value/rejection histories and 20 direct-source controls exercise neighboring + * boundaries. Effects also check exit, reentry, delivery continuity in both + * callback channels, every delivered value/previousValue, and post-disposal + * silence. Another 396 histories use the exported compiler across FROM, JOIN + * and union placement, both compilation orders and shared/separate caches at + * initial graph drain. Optimizer cloning can prevent a placement from reusing + * the original raw key; explicit counters distinguish that from cache reuse. + * It invokes real query callbacks and compares their copied input and ordinary derived output at + * initial readiness, held mutation, settlement and later remote use. The + * materializer receiver uses the same judgment, not a competing child model. + * + * Reconstruction retains every single-field collision and both populated union + * branches with numeric/string keys. Removing path, shape, collision, boolean + * or checkpoint loses a distinct sanitizer shortcut; the calibrated attacks + * below exhibit those consequences. Legal direct source returns cannot carry + * authored overrides. Compiled Collection-valued inputs are the nearby rejected + * form. Recursion is bounded at two QueryRefs, one LEFT join and flat child + * values. This establishes no native/framework or provider handoff. + * + * There is no random campaign or seed replay for this exhaustive finite grammar. + * Reach explicitly ignores unrelated seed environment variables. A calibration- + * only selection enters no receiving campaign; a partial receiving selection + * still owes the selected campaign's entire required matrix. + */ +import { afterAll, expect, test } from 'vitest' +import { declareOracle } from '../oracle-declaration.js' +import { createOracleReach } from '../oracle-reach.js' +import { + expectedMetadataInput, + expectedMetadataRecord, + expectedSourceVirtualFields, + metadataCallbackHistories, + metadataCallbackKinds, + metadataCallbackReceiverHistories, + metadataCompilerCacheHistories, + metadataInlineHistories, + metadataInputViolations, + metadataOutputViolations, +} from './virtual-row-fields-input-oracle.js' +import { + captureMetadataInput, + runInlineMetadataCallbackHistory, + runMetadataCallbackHistory, + runMetadataCompilerCacheHistory, +} from './virtual-row-fields-input-driver.js' +import type { MetadataSourceFact } from './virtual-row-fields-input-oracle.js' + +const cuts = [`initial`, `held`, `settled`, `remote-next-use`] as const +const receiverCuts = (receiver: string) => + receiver === `effect` ? [...cuts, `exit`, `reentry`, `disposed`] : cuts +const key = (history: { + shape: string + collision: string + authoredPending: boolean +}) => `${history.shape}/${history.collision}/${history.authoredPending}` +// Vitest interprets dollar-prefixed words in table titles as placeholders. +// Escape only the display label; the grammar and reach retain exact names. +const callbackCases = metadataCallbackReceiverHistories.map((history) => ({ + ...history, + label: + `${history.kind}/${history.receiver}/${history.path}/${key(history)}`.replaceAll( + `$`, + `field:`, + ), +})) +const inlineCases = metadataInlineHistories.map((history) => ({ + ...history, + label: `${history.path}/${history.form}/${key(history)}`.replaceAll( + `$`, + `field:`, + ), +})) +const compilerCases = metadataCompilerCacheHistories.map((history) => ({ + ...history, + label: + `${history.path}/${history.order}/${history.cache}/${key(history)}`.replaceAll( + `$`, + `field:`, + ), +})) +const compilerObservations = [ + `callback`, + `derived-output`, + `authored-output`, +] as const +const compilerReach = createOracleReach( + `compiler cache metadata`, + metadataCompilerCacheHistories.flatMap((history) => + compilerObservations.map( + (observation) => + `${history.path}/${history.order}/${history.cache}/${key(history)}/${observation}`, + ), + ), + { replaying: false }, +) +const callbackReach = createOracleReach( + `functional metadata callback`, + [ + ...metadataCallbackReceiverHistories.flatMap((history) => + receiverCuts(history.receiver).map( + (cut) => + `${history.kind}/${history.receiver}/${history.path}/${key(history)}/${cut}`, + ), + ), + ...metadataCallbackKinds.flatMap((kind) => + [`root`, `inline-child`, `inline-child-direct`, `effect`].flatMap( + (receiver) => + receiverCuts(receiver).map( + (cut) => `${kind}/${receiver}/direct-source/${cut}`, + ), + ), + ), + ], + { replaying: false }, +) +const inlineReach = createOracleReach( + `inline metadata callback`, + metadataInlineHistories.flatMap((history) => + history.form === `collection` + ? [`${history.path}/${history.form}/${key(history)}/rejected`] + : cuts.map( + (cut) => `${history.path}/${history.form}/${key(history)}/${cut}`, + ), + ), + { replaying: false }, +) +let callbackEntered = false +let inlineEntered = false +let compilerEntered = false +afterAll(() => { + if (callbackEntered) callbackReach.assertReached() + if (inlineEntered) inlineReach.assertReached() + if (compilerEntered) compilerReach.assertReached() + if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { + for (const [name, reach] of [ + [`callback`, callbackReach], + [`inline`, inlineReach], + [`compiler`, compilerReach], + ] as const) { + const counts = [...reach.counts().values()] + console.info(`${name} metadata reach`, { + states: counts.length, + reached: counts.filter(Boolean).length, + minimum: Math.min(...counts), + }) + } + } +}) +declareOracle({ + check: [metadataInputViolations, metadataOutputViolations], + model: [expectedMetadataInput], + grammar: [ + metadataCallbackHistories, + metadataCallbackKinds, + metadataCallbackReceiverHistories, + metadataInlineHistories, + metadataCompilerCacheHistories, + ], +}) + +test.each(callbackCases)( + `receives callback metadata before deriving ordinary values: $label`, + async (history) => { + callbackEntered = true + callbackReach.beginHistory() + const report = await runMetadataCallbackHistory( + history, + history.path, + (cut) => + callbackReach.reach( + `${history.kind}/${history.receiver}/${history.path}/${key(history)}/${cut}`, + ), + history.receiver, + history.kind, + ) + expect( + report.violations, + JSON.stringify(report.violations[0]), + ).toStrictEqual([]) + }, +) +test.each( + metadataCallbackKinds.flatMap((kind) => + ([`root`, `inline-child`, `inline-child-direct`, `effect`] as const).map( + (receiver) => ({ kind, receiver }), + ), + ), +)( + `receives direct source metadata before deriving ordinary values: %j`, + async ({ kind, receiver }) => { + callbackEntered = true + callbackReach.beginHistory() + const report = await runMetadataCallbackHistory( + { + shape: `functional-direct`, + collision: `none`, + authoredPending: false, + }, + `direct-source`, + (cut) => callbackReach.reach(`${kind}/${receiver}/direct-source/${cut}`), + receiver, + kind, + ) + expect(report.violations).toStrictEqual([]) + }, +) +test.each(inlineCases)( + `receives inline authored metadata or rejects compiled Collection input: $label`, + async (history) => { + inlineEntered = true + inlineReach.beginHistory() + const report = await runInlineMetadataCallbackHistory( + history, + history.form, + (cut) => + inlineReach.reach( + `${history.path}/${history.form}/${key(history)}/${cut}`, + ), + history.path, + ) + expect( + report.violations, + JSON.stringify(report.violations[0]), + ).toStrictEqual([]) + }, +) + +// The initial compiler graph-drain cut is distinct from the Collection +// lifecycle cuts above. Both orders must preserve the cache-transparency +// relation. Independent source/projection judgments still check both executions; +// raw output does not borrow an implicit Effect-root attribution policy. +test.each(compilerCases)( + `preserves metadata across compiler cache modes: $label`, + async (history) => { + compilerEntered = true + compilerReach.beginHistory() + const report = await runMetadataCompilerCacheHistory( + history, + history.order, + history.cache, + history.path, + (observation) => + compilerReach.reach( + `${history.path}/${history.order}/${history.cache}/${key(history)}/${observation}`, + ), + ) + expect( + report.violations, + JSON.stringify(report.violations[0]), + ).toStrictEqual([]) + }, +) + +const sourceFact: MetadataSourceFact = { + row: { id: 1, parentId: `group`, label: `before`, optional: undefined }, + sourceId: `genuine-source`, + pending: false, +} +// Controls alter observations, never source files. The allowed answer comes +// from the same judgment used for every real receiving history above. +test.each([ + `source-spoof`, + `source-stripped`, + `projection-leak`, + `inline-stripped`, + `own-undefined-lost`, + `mixed-before-after`, + `late-capture-rewrite`, +] as const)(`rejects a hostile callback-input observation: %s`, (fault) => { + const description = { + shape: `structured-spread`, + collision: `all`, + authoredPending: true, + } as const + const projected = { ...description, shape: `functional-spread` } as const + const use = fault === `projection-leak` ? projected : description + const inline = fault === `inline-stripped` + const expected = expectedMetadataInput(sourceFact, use, inline) + const hostile = structuredClone(expected) + if (fault === `source-spoof`) hostile.fields.$collectionId!.value = `spoof` + if (fault === `source-stripped` || fault === `inline-stripped`) + hostile.fields.$key = { present: false, own: false, value: undefined } + if (fault === `projection-leak`) + hostile.fields.$key = { present: true, own: true, value: `spoof` } + if (fault === `own-undefined-lost`) delete hostile.ordinary.optional + const next = expectedMetadataInput( + { + ...sourceFact, + row: { ...sourceFact.row, label: `pending` }, + pending: true, + }, + use, + inline, + ) + if (fault === `mixed-before-after`) hostile.fields = next.fields + if (fault === `late-capture-rewrite`) hostile.ordinary.label = `later` + expect(metadataInputViolations(expected, [expected])).toStrictEqual([]) + expect(metadataInputViolations(hostile, [expected, next])).not.toStrictEqual( + [], + ) +}) +test.each([ + `omitted-peer`, + `duplicate-peer`, + `wrong-key-kind`, + `publication-only-sanitizer`, + `missing-derived-field`, +] as const)(`rejects a hostile derived ordinary output: %s`, (fault) => { + const description = { + shape: `functional-projection`, + collision: `$collectionId`, + authoredPending: false, + } as const + const expected = [ + sourceFact, + { ...sourceFact, row: { ...sourceFact.row, id: `1` } }, + ].map((fact) => ({ + id: fact.row.id, + observed: expectedMetadataInput(fact, description), + })) + const hostile = structuredClone(expected) + if (fault === `omitted-peer`) hostile.pop() + if (fault === `duplicate-peer`) hostile.push(structuredClone(hostile[1]!)) + if (fault === `wrong-key-kind`) hostile[1]!.id = 1 + if (fault === `publication-only-sanitizer`) + hostile[0]!.observed.fields.$collectionId = { + present: true, + own: true, + value: `spoof`, + } + if (fault === `missing-derived-field`) + Reflect.deleteProperty(hostile[0]!, `observed`) + expect(metadataOutputViolations(expected, expected)).toStrictEqual([]) + expect(metadataOutputViolations(hostile, expected)).not.toStrictEqual([]) +}) +test(`copies callback input before later source objects change`, () => { + const description = { + shape: `functional-direct`, + collision: `none`, + authoredPending: false, + } as const + const value = { + ...sourceFact.row, + ...expectedSourceVirtualFields(sourceFact), + } + const capture = captureMetadataInput(value) + value.label = `later` + expect( + metadataInputViolations(capture, [ + expectedMetadataInput(sourceFact, description), + ]), + ).toStrictEqual([]) +}) +test(`accepts readable inherited source metadata at callback input`, () => { + const description = { + shape: `functional-direct`, + collision: `none`, + authoredPending: false, + } as const + const value: object = Object.assign( + Object.create(expectedSourceVirtualFields(sourceFact)), + sourceFact.row, + ) + const capture = captureMetadataInput(value) + const expected = expectedMetadataInput(sourceFact, description) + expect(metadataInputViolations(capture, [expected])).toStrictEqual([]) + expect( + metadataOutputViolations( + [{ id: 1, observed: capture }], + [{ id: 1, observed: expected }], + ), + ).toStrictEqual([]) +}) +test(`copies nested callback values before later child objects change`, () => { + const child = { + ...sourceFact.row, + ...expectedSourceVirtualFields(sourceFact), + } + const capture = captureMetadataInput({ id: `parent`, children: [child] }) + const expected = expectedMetadataRecord({ + id: `parent`, + children: [ + { ...sourceFact.row, ...expectedSourceVirtualFields(sourceFact) }, + ], + }) + child.label = `later` + expect(metadataInputViolations(capture, [expected])).toStrictEqual([]) +}) +test.each([`selected-virtual`, `own-undefined`] as const)( + `rejects an inherited replacement for an authored inline field: %s`, + (fault) => { + const description = { + shape: `functional-projection`, + collision: `$collectionId`, + authoredPending: false, + } as const + const selected = expectedMetadataInput(sourceFact, description, true) + const child: Record = { + ...selected.ordinary, + ...Object.fromEntries( + Object.entries(selected.fields) + .filter(([, field]) => field.present) + .map(([name, field]) => [name, field.value]), + ), + } + const expected = expectedMetadataRecord({ id: `parent`, children: [child] }) + const field = fault === `selected-virtual` ? `$collectionId` : `optional` + const inherited: object = Object.assign( + Object.create({ [field]: child[field] }), + Object.fromEntries( + Object.entries(child).filter(([name]) => name !== field), + ), + ) + const actual = captureMetadataInput({ id: `parent`, children: [inherited] }) + expect( + metadataInputViolations( + captureMetadataInput({ id: `parent`, children: [child] }), + [expected], + ), + ).toStrictEqual([]) + expect(metadataInputViolations(actual, [expected])).not.toStrictEqual([]) + expect( + metadataOutputViolations( + [{ id: `parent`, observed: actual }], + [{ id: `parent`, observed: expected }], + ), + ).not.toStrictEqual([]) + }, +) + +// The inline array is public data too. An extra ordinary or reserved own field +// must survive capture so both judgments can reject the unauthored shape. +test.each([`unexpected`, `$collectionId`] as const)( + `rejects an extra own field on an inline array: %s`, + (field) => { + const child = expectedMetadataInput( + sourceFact, + { + shape: `functional-projection`, + collision: `none`, + authoredPending: false, + }, + true, + ).ordinary + const authored = { id: `parent`, children: [child] } + const expected = expectedMetadataRecord(authored) + const value = structuredClone(authored) + const lawful = captureMetadataInput(value) + expect(metadataInputViolations(lawful, [expected])).toStrictEqual([]) + Object.assign(value.children, { [field]: `spoof` }) + const hostile = captureMetadataInput(value) + const rejected = { + input: metadataInputViolations(hostile, [expected]).length > 0, + output: + metadataOutputViolations( + [{ id: `parent`, observed: hostile }], + [{ id: `parent`, observed: expected }], + ).length > 0, + } + expect(rejected).toStrictEqual({ input: true, output: true }) + }, +) diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.ts new file mode 100644 index 0000000000..f64b6916be --- /dev/null +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.ts @@ -0,0 +1,306 @@ +/** + * # What may user JavaScript learn from a query input? + * + * Authority: the maintainer's O12 decision, recorded in the RC challenge on + * 2026-10-28, extended by the maintainer on 2026-10-30 to fn.select, fn.where + * and fn.having, and ARCHITECTURE.md's functional-input boundary. Direct source + * returns and structured source spreads expose genuine source metadata. + * Own-shape projections (including functional spreads) expose no reserved + * fields before a downstream callback. Inline include values keep authored + * shape. Collection-valued compiled inputs remain inadmissible. + * + * This small model uses authored source facts and syntax descriptions, not + * compiler provenance or observed values. Callback input is a model observation + * of one runtime value; it is not a published Collection row. An ordinary + * derived output carries this observation so publication-only sanitization + * cannot erase an earlier violation. The finite grammar bounds recursion at + * two QueryRefs and joins at one LEFT join. Provider/framework timing, arbitrary + * objects/descriptors and same-key overlapping transactions are outside it. + */ +import { isDeepStrictEqual } from 'node:util' +import { declareOracle } from '../oracle-declaration.js' +import { virtualFieldNames } from './virtual-row-fields-support.js' +import { expectedSourceMetadata } from './virtual-row-fields-source-oracle.js' +import { + metadataCollisionCases, + metadataProjectionShapes, +} from './virtual-row-fields-grammar-oracle.js' + +export type MetadataInputShape = (typeof metadataProjectionShapes)[number] +export type MetadataInputRow = { + id: string | number + parentId: string + label: string + // An explicit undefined ordinary property must survive both boundaries. + optional: undefined +} +export type MetadataSourceFact = { + row: MetadataInputRow + sourceId: string + pending: boolean +} +export type MetadataCollision = (typeof metadataCollisionCases)[number] +export type MetadataInputDescription = MetadataCollision & { + shape: MetadataInputShape +} +export const metadataInputDescriptions = metadataProjectionShapes.flatMap( + (shape) => + metadataCollisionCases + .filter( + ({ collision }) => + shape !== `functional-direct` || collision === `none`, + ) + .map((collision) => ({ shape, ...collision })), +) +export const metadataCallbackPaths = [ + `query-ref`, + `nested-query-ref`, + `joined-from`, + `joined-source`, + `union-first`, + `union-last`, + `union-query-ref`, +] as const +export type MetadataCallbackPath = (typeof metadataCallbackPaths)[number] +export const metadataCallbackHistories = metadataInputDescriptions.flatMap( + (description) => + metadataCallbackPaths.map((path) => ({ ...description, path })), +) +export const metadataCallbackKinds = [ + `select`, + `where`, + `having`, + `where-select`, + `having-select`, +] as const +export type MetadataCallbackKind = (typeof metadataCallbackKinds)[number] +export const metadataCallbackReceiverHistories = + metadataCallbackHistories.flatMap((history) => + ([`root`, `inline-child`, `inline-child-direct`, `effect`] as const) + // A flat union has no named source reference for include correlation. + // Admission requires the QueryRef used by union-query-ref or the wrapped + // receiver; direct union-first/last are not legal include children. + .filter( + (receiver) => + receiver !== `inline-child-direct` || + (history.path !== `union-first` && history.path !== `union-last`), + ) + .flatMap((receiver) => + metadataCallbackKinds.map((kind) => ({ ...history, receiver, kind })), + ), + ) +export const metadataInlineHistories = metadataInputDescriptions.flatMap( + (description) => + ([`array`, `materialized`, `collection`] as const).flatMap((form) => + ([`direct`, `query-ref`] as const).map((path) => ({ + ...description, + form, + path, + })), + ), +) +// The exported compiler also accepts an exact QueryIR and an optional cache. +// Reusing one selected query across two roots must preserve both its authored +// output and the source/projection judgment before a downstream callback. +// Separate caches distinguish that promise from an ordinary projection defect. +export const metadataCompilerCachePaths = [ + `query-ref`, + `joined-source`, + `union-branch`, +] as const +export type MetadataCompilerCachePath = + (typeof metadataCompilerCachePaths)[number] +export const metadataCompilerCacheHistories = metadataInputDescriptions.flatMap( + (description) => + ([`authored-first`, `callback-first`] as const).flatMap((order) => + ([`shared`, `separate`] as const).flatMap((cache) => + metadataCompilerCachePaths.map((path) => ({ + ...description, + order, + cache, + path, + })), + ), + ), +) +export type MetadataInputSnapshot = { + present: boolean + ordinary: Record + fields: Record +} + +// This is the existing source-row law, shared with root publication. A pending +// unconfirmed update drops at settlement; a later remote write stays remote. +export function expectedSourceVirtualFields(fact: { + row: { id: string | number } + sourceId: string + pending: boolean +}) { + return expectedSourceMetadata(fact.row.id, fact.sourceId, fact.pending) +} +export function authoredMetadataCollision( + description: MetadataCollision, +): Record { + return Object.fromEntries( + virtualFieldNames + .filter( + (field) => + description.collision === field || description.collision === `all`, + ) + .map((field) => [ + field, + field === `$hasPendingWrites` + ? description.authoredPending + : field === `$origin` + ? `local` + : `spoof-${field}`, + ]), + ) +} +function ordinaryProjection( + fact: MetadataSourceFact, + shape: MetadataInputShape, +): Record { + if (shape === `structured-projection`) + return { + id: fact.row.id, + label: fact.row.label, + hasPendingWrites: fact.pending, + } + if (shape === `functional-projection`) + return { + id: fact.row.id, + label: fact.row.label, + optional: undefined, + } + return { ...fact.row } +} +export function expectedMetadataInput( + fact: MetadataSourceFact, + description: MetadataInputDescription, + inline = false, +): MetadataInputSnapshot { + const ordinary = ordinaryProjection(fact, description.shape) + const sourceOwned = + description.shape === `structured-spread` || + description.shape === `functional-direct` + const fields = inline + ? { + ...(description.shape.endsWith(`projection`) + ? {} + : expectedSourceVirtualFields(fact)), + ...authoredMetadataCollision(description), + } + : sourceOwned + ? expectedSourceVirtualFields(fact) + : {} + return expectedMetadataRecord({ ...ordinary, ...fields }) +} +// A record snapshot is also the common judgment for inline parents. It keeps +// missing fields distinct from own undefined and checks every ordinary field. +export function expectedMetadataRecord( + record: Record | undefined, +): MetadataInputSnapshot { + return { + present: record !== undefined, + ordinary: Object.fromEntries( + Object.entries(record ?? {}).filter( + ([key]) => !virtualFieldNames.some((field) => field === key), + ), + ), + fields: Object.fromEntries( + virtualFieldNames.map((field) => [ + field, + { + present: record !== undefined && field in record, + own: record !== undefined && Object.hasOwn(record, field), + value: record?.[field], + }, + ]), + ), + } +} +// Reserved metadata promises presence and readable value, not descriptor +// layout. Keep `own` in captures as diagnostics without requiring it for a +// present field. An absent field cannot legally be own. This applies only to +// top-level callback metadata. Nested inline selected records stay in ordinary +// data and retain their authored own fields, including own undefined. +function readableMetadataInput(snapshot: MetadataInputSnapshot) { + return { + ...snapshot, + fields: Object.fromEntries( + Object.entries(snapshot.fields).map(([name, field]) => [ + name, + field.present ? { present: true, value: field.value } : field, + ]), + ), + } +} +export function metadataInputViolations( + actual: MetadataInputSnapshot, + allowed: ReadonlyArray, +): Array { + return allowed.some((expected) => + isDeepStrictEqual( + readableMetadataInput(actual), + readableMetadataInput(expected), + ), + ) + ? [] + : [`callback input differs from every permitted authored source state`] +} +export function metadataOutputViolations( + actual: ReadonlyArray<{ + id: string | number + observed: MetadataInputSnapshot + }>, + expected: ReadonlyArray<{ + id: string | number + observed: MetadataInputSnapshot + }>, +): Array { + const byId = (a: { id: string | number }, b: { id: string | number }) => + `${typeof a.id}:${a.id}`.localeCompare(`${typeof b.id}:${b.id}`) + // The derived output carries the recorder's diagnostic ownership flag too. + // Judge its metadata facts by the same readable-value law, while exact outer + // fields, typed keys, ordinary input fields and occurrence counts stay strict. + const stripDiagnostics = (rows: ReadonlyArray) => + rows.map((value) => { + if (!value || typeof value !== `object`) return value + const row = value as Record + const observed = row.observed + if ( + !observed || + typeof observed !== `object` || + !(`fields` in observed) || + !observed.fields + ) + return row + return { + ...row, + observed: readableMetadataInput(observed as MetadataInputSnapshot), + } + }) + return isDeepStrictEqual( + stripDiagnostics([...actual].sort(byId)), + stripDiagnostics([...expected].sort(byId)), + ) + ? [] + : [`derived ordinary output differs from the callback-input law`] +} + +declareOracle({ + model: [ + expectedSourceVirtualFields, + expectedMetadataInput, + expectedMetadataRecord, + ], + check: [metadataInputViolations, metadataOutputViolations], + grammar: [ + metadataInputDescriptions, + metadataCallbackHistories, + metadataCallbackReceiverHistories, + metadataInlineHistories, + metadataCompilerCacheHistories, + ], +})