From f1dc604aa3c22c72aec9964059ae8d4222cb4086 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 10:15:27 -0600 Subject: [PATCH 01/19] fix(db): sanitize recursive source metadata before callbacks --- docs/contributing/oracle-coverage.md | 48 +- .../2026-10-29-pr-2171-deferred-oracles.md | 4 + .../oracle-reviews/callback-metadata.md | 205 ++++++ packages/db/package.json | 2 +- packages/db/src/query/compiler/index.ts | 10 +- packages/db/src/query/live/ARCHITECTURE.md | 13 +- .../query/virtual-row-fields-input-driver.ts | 638 ++++++++++++++++++ .../virtual-row-fields-input-oracle.test.ts | 346 ++++++++++ .../query/virtual-row-fields-input-oracle.ts | 267 ++++++++ 9 files changed, 1502 insertions(+), 31 deletions(-) create mode 100644 docs/contributing/oracle-reviews/callback-metadata.md create mode 100644 packages/db/tests/query/virtual-row-fields-input-driver.ts create mode 100644 packages/db/tests/query/virtual-row-fields-input-oracle.test.ts create mode 100644 packages/db/tests/query/virtual-row-fields-input-oracle.ts diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 5e430487e0..69cb4470e8 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -796,27 +796,37 @@ stay ready/empty, and a new interval gets a new handle. This assumes no facade-local optimistic mutations. Every shape declares active empty readiness and nonempty child receiving cuts. Six lifetime and four inline-value hostile controls reject wrong observations. See [chunk 5](oracle-reviews/2026-10-28-pr-2171-oracle-chunks.md). -Functional callback-input observations now have a separate receiving boundary -in `virtual-row-fields-input-oracle.test.ts`, sharing source metadata judgment -with the existing root/Effect owners. Its finite 33 authoring descriptions cross -seven QueryRef/join/union consumers through root and correlated inline-child -compilation (462 histories), plus two direct-source controls. Each compares -immutable callback captures and their ordinary derived output at initial +Functional callback-input observations have a separate receiving boundary in +`virtual-row-fields-input-oracle.test.ts`, sharing source metadata judgment with +the existing root/Effect owners. Its 33 authoring descriptions cross seven +QueryRef/join/union paths through root and wrapped correlated child compilation +(462 histories), plus five direct correlated child paths (165 histories). +Flat union children require a named QueryRef for include correlation; their +supported forms remain in the wrapped and union-query-ref paths. Three +source controls preserve genuine metadata without an intermediate projection. +Each compares immutable callback captures and ordinary derived output at initial readiness, held mutation, settlement and remote next use. Complete before/after -source facts permit retraction evaluation; startup LEFT joins may expose an -absent side, while established outputs require both typed-key rows. Reserved -metadata is judged by presence/readable value, with ownership flags diagnostic; -ordinary own undefined remains distinct from absence. Another 66 histories keep +source facts permit retraction evaluation. Startup LEFT joins may expose an +absent side; established outputs require both numeric/string-key rows. Reserved +metadata is judged by presence/readable value, with ownership diagnostic. +Ordinary own undefined stays distinct from absence. Another 66 histories retain nested inline array/materialized authored values, and 33 reject compiled -Collection inputs before callbacks. Per-history/path/cut reach is mandatory even -under unrelated replay seeds; unentered calibration-only runs stand aside. -Unchanged production exposes 203 correlated-child violations: new-shape fields -leak and source-spread collisions spoof genuine metadata. Root receivers and -nested inline preservation pass. All 2,153 declared states reach; 17 controls -and independent seeded/unentered/partial-selection checks calibrate this owner. -See [chunk 17](oracle-reviews/2026-10-28-pr-2171-oracle-chunks.md) for exact -fingerprints and independent evidence. Production repair is deferred under the -[RC oracle challenge](oracle-reviews/2026-10-28-pr-2171-oracle-challenge.md). +Collection inputs before callbacks. All 2,520 callback and 297 inline/rejection +states reach once; 17 controls calibrate the same judgments. Partial receiving +selections still owe the selected campaign's full reach matrix. + +The restored owner reproduced 203 correlated-child REDs on unchanged main. +Adding the direct receiving paths before production repair raised that to 348 +REDs with 398 passes. The compiler now enforces source/projection metadata at +QueryRef and joined-source boundaries independently of final inline output. +All 746 owner tests pass. A QueryRef-only mutant leaves 29 joined-source +violations; sanitizing every inline child breaks 58 authored-value histories. +The [runtime repair record](oracle-reviews/callback-metadata.md) retains the +commands, distinguishing traces, source fingerprints and limits. Historical +instrument acceptance remains in +[chunk 17](oracle-reviews/2026-10-28-pr-2171-oracle-chunks.md). +The separate static receiving-type failures in the +[deferred record](oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md) remain open. This extension does not establish arbitrary descriptors/prototypes, greater recursive depth, same-source overlapping transactions, other dynamic join kinds, truncate, provider scheduling, framework receivers or construction diff --git a/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md b/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md index c92603d698..21fb8c6fb0 100644 --- a/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md +++ b/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md @@ -19,6 +19,10 @@ No follow-up PR is claimed until one is actually created. callbacks. The approved callback law remains normative. The accepted runtime owner has 377 passes / 203 intended REDs and all 2,153 cells; the bounded concrete lane has the same seven failures and one compatible pass on main. + The runtime follow-up restores this owner and adds direct child receivers. + It reproduces 348 REDs before repair and passes all 746 tests afterward, with + all 2,817 states reached. See [the runtime repair](callback-metadata.md). + This does not close the separate static metadata receiving-type failures below. - P11/P12: raw undefined becomes null and selected empty records disappear. Their root/QueryRef runtime owner retains 18 REDs, correlated receiver 72 REDs, and source-public type receiver 280 REDs. Every declared receiving cell and diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md new file mode 100644 index 0000000000..9524882cf2 --- /dev/null +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -0,0 +1,205 @@ +# Callback metadata runtime repair + +The accepted callback-input law applies before user JavaScript derives a value. +Direct source returns and structured whole-source spreads expose genuine +`$key`, `$origin`, `$hasPendingWrites` and `$collectionId`. New projected shapes, +including functional spreads, expose none of these reserved fields to downstream +callbacks. Inline include values retain their authored shape. Compiled +Collection-valued callback inputs reject before user code runs. + +Authority: the maintainer's O12 decision from 2026-10-28 and the +[architecture's callback-input boundary](../../../packages/db/src/query/live/ARCHITECTURE.md#route-context-transport). +This repair enforces that existing runtime law. It revises no product contract. +The [coverage owner](../oracle-coverage.md) retains the separate static type gaps. + +## Independent owner and receiving histories + +The restored owner consists of the +[model and grammar](../../../packages/db/tests/query/virtual-row-fields-input-oracle.ts), +[production driver](../../../packages/db/tests/query/virtual-row-fields-input-driver.ts) +and [runner and controls](../../../packages/db/tests/query/virtual-row-fields-input-oracle.test.ts). +It derives expectations from authored source facts and projection syntax. +It shares the source metadata law with the existing publication owner. +It imports no compiler metadata classifier for its expected result. + +Thirty-three legal shape/collision descriptions cover five projection forms, +no collision, each reserved name, all names, and both authored pending booleans. +Direct source returns admit no authored override. Seven QueryRef/join/union +paths run at root and through a wrapped correlated child: 231 histories each. +Five paths also run directly as the correlated child: 165 histories. Flat union +children have no named source reference for include correlation, so their +supported forms use the wrapper or union-query-ref path. They remain covered. +Three direct-source controls, 66 inline array/materialized histories, 33 compiled +Collection rejections and 17 checker/recorder controls complete 746 tests. + +Every receiving history observes initial readiness, a held optimistic update, +settlement without confirmation, and a later remote update. Source rows have +numeric `1` and string `"1"` keys and distinct labels. Each callback copies its +input immediately, then returns that capture as ordinary output. The comparison +checks every callback against coherent before/after source facts and checks the +complete derived output at each established cut. Retractions may read the old +row. Startup LEFT joins may expose an absent joined side. Neither exception +permits mixed old/new fields or missing peers at the established output cut. + +Reserved fields promise readable presence/value, without a descriptor layout. +Ordinary own undefined and authored nested inline properties retain exact shape. +Failures accumulate through later cuts. Cleanup preserves the primary mismatch +and attempts every resource release. The model retains source facts, not a +second compiler or query queue. + +## RED before production edits + +The production baseline was `c351fc04d7f70dc59bf92ecbed310f899b9511cd`. +The three original owner files were restored from accepted source tree +`db38b4e009146af53e46aeeef3206269f5968af8` before any compiler edit. +The original 580 tests reproduced 203 correlated-child failures and 377 passes. +All 2,153 original reach states reached once. + +The direct child receiver was then added on unchanged production. The complete +746-test owner produced 348 comparison failures and 398 passes. Root controls +and inline authored values passed. Wrapped children contributed 203 failures; +direct children contributed 145. All 2,520 callback and 297 inline/rejection +states reached once, including later cuts after the first violation. + +Two representative distinguishing traces are retained by the executable grammar: + +- `inline-child/query-ref/functional-projection/$collectionId/false`, initial: + the projected callback input exposed authored `$collectionId: "spoof-$collectionId"`. + The model requires that field to be absent. The ordinary derived capture + preserved this earlier violation through final publication. +- `inline-child-direct/joined-source/structured-spread/all/true`, initial: + the callback input exposed authored pending `true`, origin `local`, and spoofed + key/Collection ID. The genuine source was remote, pending `false`, with numeric + key `1` or string key `"1"` and Collection ID `callback-first`. + +The first setup attempt lacked `fake-indexeddb/auto` because the new worktree +had no package dependency link. It ran zero tests and supplies no RED evidence. +Early direct flat-union fixtures failed correlation admission. The legal grammar +uses a named QueryRef for those unions. Neither setup failure counts as a defect. +No comparison, reach minimum or run budget was weakened to obtain GREEN. + +## Repair and discriminators + +A recursive QueryRef or joined subquery exports a source value even when its +consumer is inside an inline include. Those source boundaries now use the +existing source/projection metadata mode. The enclosing include's final value +keeps the existing inline mode. The obsolete `processFrom` mode parameter is +removed. No new metadata registry, state, traversal or public API is added. + +All 746 tests pass with all 2,817 reach states at count one. Seven neighboring +suites pass 1,883 tests: root metadata, functional projection/input rejection, +route-context transport, public container copying, include query shapes and +subquery user values. Focused no-emit TypeScript, ESLint and changed-file oracle +lint pass. After integrating #2301 (`bfd1da1a7e752d66fd667adfd3cdf54f60732b22`), +the normal package pnpm configuration also passes all 746 tests and all reach. +No local build or dependency installation ran. + +Two deliberately wrong repairs ran in copied source trees, without changing the +reviewed worktree: + +| Mutant | Intended wrong design | Result at the shared comparison | +| --- | --- | --- | +| QueryRef-only | Keep the old joined-source mode | 29 failures, 717 passes | +| Sanitize inline | Sanitize every final child instead of preserving authored inline values | 58 failures, 688 passes | + +Both mutants reached the intended callback/output comparisons. They are assertion +failures, not timeouts or setup errors. The original 348-failure RED is the +original-defect discriminator. The direct receiver prevents an outer QueryRef +from concealing an unrepaired joined-source boundary. + +## Prep review and loss audit + +Separate read-only review and simplification passes found no confirmed production +or oracle-enforcement defect. Both found stale opening counts. Those counts are +corrected. The review also required these planned coverage/architecture updates. + +A proposed shared-query cache conflict was tested through actual Effects and +live-query Collections. Each receiver reused one projected query through a +QueryRef, then through both union branch orders. All six probes passed: the +nested callback saw no `$collectionId`. The direct Effect branch retained its +authored reserved output, which is separate from downstream callback input. +The initial Collection-union fixture supplied conflicting custom public keys; +using the compiler's union keys corrected the fixture. Those initial failures +establish no product violation. + +The probe constructs the following source and nested consumer before selecting +QueryRef-only, `unionAll(projected, consumed)`, or its reverse: + +```ts +const projected = new Query().from({ source }).select(({ source }) => ({ + id: source.id, + $collectionId: 'spoof', +})) +const consumed = new Query().from({ nested: projected }).fn.select(({ nested }) => { + captures.push({ present: '$collectionId' in nested, value: nested.$collectionId }) + return { id: nested.id, observed: nested.$collectionId } +}) +``` + +Each of the six runs entered the callback once and captured +`{ present: false, value: undefined }`. This refutes that concrete cache +hypothesis. It is no universal cache proof or generated Effect lifecycle claim. +The source-level review distinguished final inline/Effect values from callback +input. Its finite probes supply useful cross-path evidence. The reviewers did +not independently rerun the full owner or mutation campaign. + +Four retained items reconcile as two fixed documentation items, one duplicate +count correction and one refuted cache hypothesis. Both review passes are useful +for this bounded patch. Neither supplies a whole-library correctness verdict. + +## Rubric accounting and limits + +ORC-001–005: accepted authority, independent source facts, literate layers, +finite legal grammar, real callback captures and exact derived output cuts. +Removing receiver, path, collision, pending boolean or checkpoint loses a +named source/projection/inline distinction. Both populated union positions and +both typed keys remain represented. Direct-source controls distinguish a +blanket stripping rule. The rejected flat-union form bounds admission. + +ORC-006: 17 hostile observation/recorder controls and both production mutants. +ORC-007: exhaustive finite enumeration, so random campaigns and seed replay do +not apply. A valid unrelated `utils.1` seed/path run passes all 746 tests with +all reach intact. A calibration-only selection passes 17 controls without +entering a receiver. Selecting only the three direct-source controls fails the +missing-matrix reach assertion. An omitted replay property was a configuration +error with zero tests, not product evidence. ORC-008–009: +source facts distinguish coherent retractions; callback input is distinct from +published row, with that abstraction stated beside the model. ORC-010: all-cut +failure reports and failure-preserving cleanup. ORC-011: the concrete shared-cache +hypothesis has six same-path probes; no remaining shared-model hypothesis was +identified. ORC-012: this versioned record and source fingerprints below. +ORC-013: single/all/no collisions, typed peers, source/new/inline forms and the +two mutants distinguish plausible shortcut repairs. ORC-014: the declared +boundary is core runtime compilation, with no real-provider/framework timing +claim. ORC-015: every named callback and inline/rejection state reaches once. +ORC-016: all histories and controls use the shared independent judgments. + +The runtime evidence is finite: flat inline children, at most two QueryRefs, +one LEFT join, two typed-key peers, one held successful mutation, and four cuts. +Arbitrary descriptors/prototypes, deeper recursion, same-key overlapping writes, +other join kinds, truncate, framework/provider scheduling and construction-failure +cleanup remain outside it. The deferred static metadata receivers and literal +value/type defects remain open in the [retained record](2026-10-29-pr-2171-deferred-oracles.md). +This runtime repair does not claim complete static or P11/P12 closure. + +## Reproduction and reviewed sources + +From the repository root, run this targeted owner without a build or install: + +```sh +TANSTACK_DB_ORACLE_STATISTICS=1 pnpm_config_verify_deps_before_run=false \ + pnpm --dir packages/db exec vitest run --maxWorkers=2 \ + --coverage.enabled=false --typecheck.enabled=false \ + tests/query/virtual-row-fields-input-oracle.test.ts +``` + +The source fingerprints identify the reviewed implementation independently of +later prose edits. The parent baseline and exact legal grammar reproduce the +original failures without temporary log files. + +| Source | SHA256 | +| --- | --- | +| `packages/db/src/query/compiler/index.ts` | `3982fc56163cfc44583a737b9bbf91c5aece9d6c0212324721309646a9eb488e` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.ts` | `99a7357c283731adc7e7552a9e66e4b3d4df0df58576db82f4d9e5f0ef9f9265` | +| `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `31a19f9988ca46b82eba13b434f8d3e1deb5ff972b7fe63df64cbb8898693b79` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `2d417011231e109a9ed5194f417063bc41ac46cc36ec6405aa61e8b60408cb6b` | diff --git a/packages/db/package.json b/packages/db/package.json index 1b8a299d76..02ebf41d16 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -23,7 +23,7 @@ "test": "vitest --run", "test:dist": "vitest run --config vitest.dist.config.ts", "test:facade-retention": "node --expose-gc --import tsx tests/facade-retention.probe.ts", - "test:oracles": "vitest --run --coverage.enabled=false tests/reverse-index-oracle.property.test.ts tests/query/compiler/in-evaluator-oracle.property.test.ts tests/cleanup-queue-oracle.property.test.ts tests/comparison-oracle.property.test.ts tests/duration-query-equality-oracle.test.ts tests/cursor-oracle.property.test.ts tests/index-update-oracle.property.test.ts tests/utils-oracle.property.test.ts tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/btree-map-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/local-storage-order-oracle.test.ts tests/local-storage-value-fidelity-oracle.property.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/collection-gc-transaction-oracle.property.test.ts tests/collection-gc-retention-oracle.property.test.ts tests/collection-gc-mutation-oracle.property.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/transaction-ownership-oracle.property.test.ts tests/optimistic-settlement-boundaries-oracle.test.ts tests/optimistic-history-publication-oracle.test.ts tests/optimistic-history-outcomes-oracle.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-index-value-oracle.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-subscription-lifecycle-history-oracle.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication-oracle.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-cache-options-oracle.test.ts tests/mutation-confirmation-oracle.test.ts tests/mutation-handler-types-oracle.test.ts tests/live-query-observer-history-oracle.property.test.ts tests/live-query-observer-hydration-oracle.property.test.ts tests/live-query-observer-snapshot-oracle.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/custom-index-order-selection-oracle.test.ts tests/query/mixed-index-order-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-alias-shadowing-oracle.test.ts tests/query/includes-outer-join-route-work-oracle.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state-oracle.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/virtual-row-fields-effect-oracle.test.ts tests/query/virtual-row-fields-types-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts tests/query/pooled-live-query-oracle.property.test.ts tests/live-query-deferred-acquisition-oracle.test.ts tests/flat-change-tracking-oracle.property.test.ts tests/truncate-readiness-oracle.test.ts tests/proxy-detachment-contract-oracle.test.ts tests/production-error-messages-oracle.test.ts tests/query/bucket-facade-rollback-oracle.property.test.ts tests/query/aggregate-select-oracle.property.test.ts tests/query/load-subset-joint-replay-oracle.property.test.ts", + "test:oracles": "vitest --run --coverage.enabled=false tests/reverse-index-oracle.property.test.ts tests/query/compiler/in-evaluator-oracle.property.test.ts tests/cleanup-queue-oracle.property.test.ts tests/comparison-oracle.property.test.ts tests/duration-query-equality-oracle.test.ts tests/cursor-oracle.property.test.ts tests/index-update-oracle.property.test.ts tests/utils-oracle.property.test.ts tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/btree-map-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/local-storage-order-oracle.test.ts tests/local-storage-value-fidelity-oracle.property.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/collection-gc-transaction-oracle.property.test.ts tests/collection-gc-retention-oracle.property.test.ts tests/collection-gc-mutation-oracle.property.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/transaction-ownership-oracle.property.test.ts tests/optimistic-settlement-boundaries-oracle.test.ts tests/optimistic-history-publication-oracle.test.ts tests/optimistic-history-outcomes-oracle.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-index-value-oracle.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-subscription-lifecycle-history-oracle.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication-oracle.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-cache-options-oracle.test.ts tests/mutation-confirmation-oracle.test.ts tests/mutation-handler-types-oracle.test.ts tests/live-query-observer-history-oracle.property.test.ts tests/live-query-observer-hydration-oracle.property.test.ts tests/live-query-observer-snapshot-oracle.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/custom-index-order-selection-oracle.test.ts tests/query/mixed-index-order-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-alias-shadowing-oracle.test.ts tests/query/includes-outer-join-route-work-oracle.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state-oracle.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/virtual-row-fields-input-oracle.test.ts tests/query/virtual-row-fields-effect-oracle.test.ts tests/query/virtual-row-fields-types-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts tests/query/pooled-live-query-oracle.property.test.ts tests/live-query-deferred-acquisition-oracle.test.ts tests/flat-change-tracking-oracle.property.test.ts tests/truncate-readiness-oracle.test.ts tests/proxy-detachment-contract-oracle.test.ts tests/production-error-messages-oracle.test.ts tests/query/bucket-facade-rollback-oracle.property.test.ts tests/query/aggregate-select-oracle.property.test.ts tests/query/load-subset-joint-replay-oracle.property.test.ts", "bench:nested-includes": "vitest bench tests/query/includes-performance.bench.ts --run", "test:indexed-db:package": "pnpm --filter @tanstack/db... build && vitest run --config vitest.indexed-db.package.config.ts", "test:collection-gc:browser": "playwright test --config playwright.collection-gc.config.ts", diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 2fe86d06ad..80547a6f07 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -622,7 +622,8 @@ function compileQueryInternal( ignoreWindow, rawQuery, query.from, - (...args) => compileQueryInternal(args, true, publishCollectionRoot), + // Joined subqueries are callback inputs even inside inline includes. + (...args) => compileQueryInternal(args, true, true), aliasToCollectionId, aliasRemapping, sourceWhereClauses, @@ -1646,7 +1647,6 @@ function processFromClause( aliasRemapping, sourceWhereClauses, parentKeyStream, - publishCollectionRoot, ) return { @@ -1693,7 +1693,6 @@ function processFromClause( aliasRemapping, sourceWhereClauses, parentKeyStream, - publishCollectionRoot, ) if (!mainAlias) { @@ -1935,7 +1934,6 @@ function processFrom( aliasRemapping: Record, sourceWhereClauses: Map>, parentKeyStream?: KeyedStream, - publishCollectionRoot = false, ): { alias: string input: KeyedStream @@ -1989,7 +1987,9 @@ function processFrom( undefined, ], true, - publishCollectionRoot, + // A QueryRef exports a source row, not its enclosing inline value. + // Enforce source/projection metadata before any consumer runs. + true, ) // Pull up alias mappings from subquery to parent scope. diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 6dc031ae33..9d78fdadf3 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -396,12 +396,13 @@ authored selected shape. Sanitizing only the eventual Collection publication cannot satisfy this law, because user code may already have derived an ordinary field from a spoofed input. -Known implementation gap for the first RC: correlated-child functional callback -inputs can still expose reserved fields from a new shape or authored collisions -from a structured source spread. The root callback receiving path passes its -bounded checks. The callback-input law above remains accepted; final publication -sanitization does not close this earlier boundary. The separate runtime/type -owners and exact RED witnesses are preserved in the +The compiler enforces this source/projection boundary when a QueryRef or joined +subquery exports its result, including inside a correlated inline child. The +include's final authored value retains its separate inline shape. The runtime +owner checks root, wrapped child and direct child callbacks at initial readiness, +held mutation, settlement and remote next use. Its bounded repair evidence is in +[the callback metadata record](../../../../../docs/contributing/oracle-reviews/callback-metadata.md). +Separate static receiving-type gaps remain in the [deferred oracle record](../../../../../docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md). D2 hashes enumerable symbol keys and uses exact local-symbol identity plus diff --git a/packages/db/tests/query/virtual-row-fields-input-driver.ts b/packages/db/tests/query/virtual-row-fields-input-driver.ts new file mode 100644 index 0000000000..a6f7b3ab57 --- /dev/null +++ b/packages/db/tests/query/virtual-row-fields-input-driver.ts @@ -0,0 +1,638 @@ +/** + * Production receiver for the callback-input oracle. Callbacks copy their facts + * immediately, including missing versus own undefined. At an action, D2 may + * invoke a callback for old-row retractions as well as new-row contributions; + * the check permits those two independently authored states. Startup LEFT joins + * may expose an absent joined side. Established output cuts require both rows. + * No callback count, ordering, or fully matched startup snapshot is promised. + */ +import { inspect } from 'node:util' +import { createCollection } from '../../src/collection/index.js' +import { + Query, + createLiveQueryCollection, + eq, + include, + materialize, + toArray, +} from '../../src/query/index.js' +import { mockSyncCollectionOptions, withOracleCleanup } from '../utils.js' +import { virtualFieldNames } from './virtual-row-fields-support.js' +import { + authoredMetadataCollision, + expectedMetadataInput, + expectedMetadataRecord, + metadataInputViolations, + metadataOutputViolations, +} from './virtual-row-fields-input-oracle.js' +import type { Collection } from '../../src/collection/index.js' +import type { QueryBuilder } from '../../src/query/builder/index.js' +import type { RefLeaf } from '../../src/query/builder/types.js' +import type { + MetadataCallbackPath, + MetadataInputDescription, + MetadataInputRow, + MetadataInputSnapshot, + MetadataSourceFact, +} from './virtual-row-fields-input-oracle.js' + +type Output = { id: string | number; observed: MetadataInputSnapshot } +type OutputContext = { + schema: {} + baseSchema: {} + fromSourceName: string + result: Output + hasResult: true +} +type ProjectionContext = { + schema: {} + baseSchema: {} + fromSourceName: string + result: MetadataInputRow + hasResult: true +} +export type CallbackInputCut = + `initial` | `held` | `settled` | `remote-next-use` +export type CallbackInputReport = { + violations: Array<{ + cut: CallbackInputCut + kind: string + actual: unknown + expected: unknown + }> + callbackCounts: Record +} + +// Throw only after every cut. Keeping the full report on the primary failure +// lets withOracleCleanup preserve it if resource release also fails. +class MetadataCallbackOracleFailure extends Error { + constructor(readonly report: CallbackInputReport) { + super( + `Callback-input law: ${inspect(report.violations[0], { depth: null })}`, + ) + this.name = `MetadataCallbackOracleFailure` + } +} + +function copyData(value: unknown): unknown { + if (Array.isArray(value)) return value.map(copyData) + if (value && typeof value === `object`) + return Object.fromEntries( + Object.getOwnPropertyNames(value).map((key) => [ + key, + copyData((value as Record)[key]), + ]), + ) + return value +} +// This recorder reads the actual public callback value. Its expected counterpart +// is authored from source facts and projection syntax in the oracle module. +// Top-level reserved fields are read through their public properties; nested +// inline records retain exact selected own-field shape through copyData. +export function captureMetadataInput(value: unknown): MetadataInputSnapshot { + const record = + value !== null && typeof value === `object` + ? (value as Record) + : undefined + return { + present: record !== undefined, + ordinary: Object.fromEntries( + Object.getOwnPropertyNames(record ?? {}) + .filter((key) => !virtualFieldNames.some((field) => field === key)) + .map((key) => [key, copyData(record![key])]), + ), + fields: Object.fromEntries( + virtualFieldNames.map((field) => [ + field, + { + present: record !== undefined && field in record, + own: record !== undefined && Object.hasOwn(record, field), + value: copyData(record?.[field]), + }, + ]), + ), + } +} + +function projectInput( + collection: Collection, + alias: string, + description: MetadataInputDescription, + parentGroup?: RefLeaf, +) { + const all = new Query().from({ [alias]: collection }) + const base = + parentGroup === undefined + ? all + : all.where((refs) => eq(refs[alias]!.parentId, parentGroup)) + const overrides = authoredMetadataCollision(description) + const shape = description.shape + const selected = + shape === `functional-direct` + ? base.fn.select((refs) => refs[alias]!) + : shape === `functional-spread` + ? base.fn.select((refs) => ({ ...refs[alias]!, ...overrides })) + : shape === `functional-projection` + ? base.fn.select((refs) => ({ + id: refs[alias]!.id, + label: refs[alias]!.label, + optional: undefined, + ...overrides, + })) + : shape === `structured-spread` + ? base.select((refs) => ({ ...refs[alias]!, ...overrides })) + : base.select((refs) => ({ + id: refs[alias]!.id, + label: refs[alias]!.label, + hasPendingWrites: refs[alias]!.$hasPendingWrites, + ...overrides, + })) + // Context erasure changes only TS's union of authoring forms. The production + // query and exact runtime fields remain checked, without casting observations. + return selected as unknown as QueryBuilder +} +function freshRows(): Array { + return [ + { id: 1, parentId: `group`, label: `number-before`, optional: undefined }, + { id: `1`, parentId: `group`, label: `string-before`, optional: undefined }, + ] +} +function source(name: string, rows: ReadonlyArray) { + return createCollection( + mockSyncCollectionOptions({ + id: name, + getKey: (row) => row.id, + initialData: rows.map((row) => ({ ...row })), + }), + ) +} +function outputRows(rows: ReadonlyArray): Array { + return rows.map( + (row) => + Object.fromEntries( + Object.entries(row).filter( + ([key]) => !virtualFieldNames.some((field) => field === key), + ), + ) as Output, + ) +} +function rowsFromSnapshots( + snapshots: ReadonlyArray, +): Array { + return snapshots.map((snapshot) => ({ + ...snapshot.ordinary, + ...Object.fromEntries( + Object.entries(snapshot.fields) + .filter(([, field]) => field.present) + .map(([name, field]) => [name, field.value]), + ), + })) +} + +/** Every history uses actual source writes and a real held mutation handler. + * Comparisons accumulate so a known initial RED cannot hide later reach. */ +export async function runMetadataCallbackHistory( + description: MetadataInputDescription, + path: MetadataCallbackPath | `direct-source`, + reached: (cut: CallbackInputCut) => void, + receiver: `root` | `inline-child` | `inline-child-direct` = `root`, +): Promise { + const initial = freshRows() + const union = path.startsWith(`union`) + const first = source(`callback-first`, union ? [initial[0]!] : initial) + const second = source(`callback-second`, [initial[1]!]) + const peers = source(`callback-peer`, initial) + const containingParents = source(`callback-containing-parent`, initial) + let facts: Array = initial.map((row, index) => ({ + row: { ...row }, + sourceId: union && index === 1 ? second.id : first.id, + pending: false, + })) + let cut: CallbackInputCut = `initial` + let allowed = facts.map((fact) => expectedMetadataInput(fact, description)) + const captures: Array<{ + cut: CallbackInputCut + input: MetadataInputSnapshot + }> = [] + const project = (row: unknown, id: string | number) => { + const observed = captureMetadataInput(row) + captures.push({ cut, input: observed }) + return { id, observed } + } + const makeQuery = (parentId?: RefLeaf) => { + const input = projectInput(first, `chosen`, description) + if (path === `direct-source`) { + const direct = new Query().from({ direct: first }) + return ( + parentId === undefined + ? direct + : direct.where(({ direct: row }) => eq(row.id, parentId)) + ).fn.select(({ direct: row }) => project(row, row.id)) + } + if (union) { + const other = projectInput(second, `other`, description) + const combined = + path === `union-last` + ? new Query().unionAll(other, input) + : new Query().unionAll(input, other) + if (path === `union-query-ref`) { + const outer = new Query().from({ nested: combined }) + return ( + parentId === undefined + ? outer + : outer.where(({ nested }) => eq(nested.id, parentId)) + ).fn.select(({ nested }) => project(nested, nested.id)) + } + return ( + parentId === undefined + ? combined + : combined.where((row) => eq(row.id, parentId)) + ).fn.select((row) => project(row, row.id)) + } + if (path === `joined-source`) { + const joined = new Query() + .from({ peer: peers }) + .leftJoin({ nested: input }, ({ peer, nested }) => + eq(peer.id, nested.id), + ) + return ( + parentId === undefined + ? joined + : joined.where(({ peer }) => eq(peer.id, parentId)) + ).fn.select(({ peer, nested }) => project(nested, peer.id)) + } + if (path === `joined-from`) { + const joined = new Query() + .from({ nested: input }) + .leftJoin({ peer: peers }, ({ nested, peer }) => eq(nested.id, peer.id)) + return ( + parentId === undefined + ? joined + : joined.where(({ nested }) => eq(nested.id, parentId)) + ).fn.select(({ nested }) => project(nested, nested.id)) + } + const once = new Query().from({ nested: input }) + if (path === `nested-query-ref`) { + const outer = new Query().from({ outer: once }) + return ( + parentId === undefined + ? outer + : outer.where(({ outer: row }) => eq(row.id, parentId)) + ).fn.select(({ outer: row }) => project(row, row.id)) + } + return ( + parentId === undefined + ? once + : once.where(({ nested }) => eq(nested.id, parentId)) + ).fn.select(({ nested }) => project(nested, nested.id)) + } + const query = + receiver === `root` + ? makeQuery() + : new Query() + .from({ parent: containingParents }) + .select(({ parent }) => ({ + id: parent.id, + // The direct receiver avoids an extra QueryRef that could sanitize + // the entire child recursively and conceal a missed union boundary. + children: + receiver === `inline-child-direct` + ? toArray( + makeQuery( + parent.id, + ) as unknown as QueryBuilder, + ) + : toArray( + new Query() + .from({ + result: + makeQuery() as unknown as QueryBuilder, + }) + .where(({ result }) => eq(result.id, parent.id)), + ), + })) + const live = createLiveQueryCollection({ + query: () => query as unknown as QueryBuilder, + getKey: (row: Output) => row.id, + }) + const report: CallbackInputReport = { + violations: [], + callbackCounts: { initial: 0, held: 0, settled: 0, 'remote-next-use': 0 }, + } + const check = () => { + const premise = + live.isReady() && first.get(1)?.$hasPendingWrites === (cut === `held`) + if (!premise) + report.violations.push({ + cut, + kind: `readiness or held-source premise missing`, + actual: { + ready: live.isReady(), + pending: first.get(1)?.$hasPendingWrites, + }, + expected: { ready: true, pending: cut === `held` }, + }) + const current = captures.filter((capture) => capture.cut === cut) + report.callbackCounts[cut] = current.length + if (current.length === 0) + report.violations.push({ + cut, + kind: `callback not entered`, + actual: current, + expected: `one or more`, + }) + for (const capture of current) { + const permitted = + cut === `initial` && path === `joined-source` + ? [...allowed, expectedMetadataRecord(undefined)] + : allowed + for (const kind of metadataInputViolations(capture.input, permitted)) + report.violations.push({ + cut, + kind, + actual: capture.input, + expected: permitted, + }) + } + const expected = facts.map((fact) => ({ + id: fact.row.id, + observed: expectedMetadataInput(fact, description), + })) + const raw = + receiver === `root` + ? live.toArray + : live.toArray.flatMap( + (row) => (row as unknown as { children: Array }).children, + ) + const actual = outputRows(raw) + for (const kind of metadataOutputViolations(actual, expected)) + report.violations.push({ cut, kind, actual, expected }) + // Reach records the actual callback path and established source/output + // boundary even when the law comparison is RED. It is not a passing count. + if (premise && current.length > 0 && raw.length === facts.length) + reached(cut) + } + let finish = async () => {} + await withOracleCleanup(async () => { + await live.preload() + check() + cut = `held` + const before = facts.map((fact) => expectedMetadataInput(fact, description)) + facts = facts.map((fact) => + fact.row.id === 1 + ? { + ...fact, + row: { ...fact.row, label: `number-pending` }, + pending: true, + } + : fact, + ) + allowed = [ + ...before, + ...facts.map((fact) => expectedMetadataInput(fact, description)), + ] + const transaction = first.update(1, (row) => { + row.label = `number-pending` + }) + finish = async () => { + first.utils.resolveSync() + await transaction.when(`settled`) + finish = async () => {} + } + check() + cut = `settled` + const pending = facts.map((fact) => + expectedMetadataInput(fact, description), + ) + facts = facts.map((fact) => + fact.row.id === 1 + ? { ...fact, row: { ...initial[0]! }, pending: false } + : fact, + ) + allowed = [ + ...pending, + ...facts.map((fact) => expectedMetadataInput(fact, description)), + ] + await finish() + check() + cut = `remote-next-use` + const settled = facts.map((fact) => + expectedMetadataInput(fact, description), + ) + facts = facts.map((fact) => + fact.row.id === `1` + ? { ...fact, row: { ...fact.row, label: `string-remote` } } + : fact, + ) + allowed = [ + ...settled, + ...facts.map((fact) => expectedMetadataInput(fact, description)), + ] + const receivingSource = union ? second : first + receivingSource.utils.begin() + receivingSource.utils.write({ type: `update`, value: { ...facts[1]!.row } }) + receivingSource.utils.commit() + check() + if (report.violations.length) + throw new MetadataCallbackOracleFailure(report) + }, [ + () => finish(), + () => live.cleanup(), + () => containingParents.cleanup(), + () => peers.cleanup(), + () => second.cleanup(), + () => first.cleanup(), + ]) + return report +} + +/** The inline receiver exercises the real materializer with authored child + * values. The same source/input judgment predicts them; no second metadata + * model is hidden in a receiving test. Compiled Collections reject at admission. */ +export async function runInlineMetadataCallbackHistory( + description: MetadataInputDescription, + form: `array` | `materialized` | `collection`, + reached: (cut: CallbackInputCut | `rejected`) => void, +): Promise { + const initial = freshRows() + const children = source(`callback-inline-child`, initial) + const parents = source(`callback-inline-parent`, [ + { id: `parent`, parentId: `group`, label: `parent`, optional: undefined }, + ]) + let facts = initial.map((row) => ({ + row: { ...row }, + sourceId: children.id, + pending: false, + })) + let cut: CallbackInputCut = `initial` + const expected = () => + expectedMetadataRecord({ + id: `parent`, + children: rowsFromSnapshots( + facts.map((fact) => expectedMetadataInput(fact, description, true)), + ), + }) + let allowed = [expected()] + // The controlled source publishes its initial child batch atomically. A + // startup callback can precede that batch or consume its complete rows. + allowed.push(expectedMetadataRecord({ id: `parent`, children: [] })) + const calls: Array<{ cut: CallbackInputCut; input: MetadataInputSnapshot }> = + [] + const build = () => + createLiveQueryCollection({ + query: (q) => { + const included = q.from({ parent: parents }).select(({ parent }) => { + const selected = projectInput( + children, + `child`, + description, + parent.parentId, + ).orderBy(({ $selected }) => $selected.label) + return { + id: parent.id, + children: + form === `collection` + ? include(selected) + : form === `array` + ? toArray(selected) + : materialize(selected), + } + }) + return q.from({ nested: included }).fn.select(({ nested }) => { + const observed = captureMetadataInput(nested) + calls.push({ cut, input: observed }) + return { id: nested.id, observed } + }) + }, + getKey: (row: Output) => row.id, + }) + const report: CallbackInputReport = { + violations: [], + callbackCounts: { initial: 0, held: 0, settled: 0, 'remote-next-use': 0 }, + } + if (form === `collection`) { + let unexpected: ReturnType | undefined + await withOracleCleanup(() => { + let error: unknown + try { + unexpected = build() + } catch (caught) { + error = caught + } + if ( + !(error instanceof Error) || + !error.message.includes( + `fn.select() cannot consume Collection-valued includes`, + ) || + calls.length > 0 + ) + report.violations.push({ + cut, + kind: `compiled Collection input admitted or callback entered`, + actual: { error, calls }, + expected: `rejection before callback`, + }) + else reached(`rejected`) + if (report.violations.length) + throw new MetadataCallbackOracleFailure(report) + }, [ + () => unexpected?.cleanup(), + () => parents.cleanup(), + () => children.cleanup(), + ]) + return report + } + const live = build() + const check = () => { + const premise = + live.isReady() && children.get(1)?.$hasPendingWrites === (cut === `held`) + if (!premise) + report.violations.push({ + cut, + kind: `inline readiness or held-source premise missing`, + actual: { + ready: live.isReady(), + pending: children.get(1)?.$hasPendingWrites, + }, + expected: { ready: true, pending: cut === `held` }, + }) + const current = calls.filter((call) => call.cut === cut) + report.callbackCounts[cut] = current.length + if (!current.length) + report.violations.push({ + cut, + kind: `inline callback not entered`, + actual: current, + expected: `one or more`, + }) + for (const call of current) + for (const kind of metadataInputViolations(call.input, allowed)) + report.violations.push({ + cut, + kind, + actual: call.input, + expected: allowed, + }) + const actual = outputRows(live.toArray) + const next = [{ id: `parent`, observed: expected() }] + for (const kind of metadataOutputViolations(actual, next)) + report.violations.push({ cut, kind, actual, expected: next }) + if (premise && current.length && live.toArray.length === 1) reached(cut) + } + let finish = async () => {} + await withOracleCleanup(async () => { + await live.preload() + check() + cut = `held` + const before = expected() + facts = facts.map((fact) => + fact.row.id === 1 + ? { + ...fact, + row: { ...fact.row, label: `number-pending` }, + pending: true, + } + : fact, + ) + allowed = [before, expected()] + const transaction = children.update(1, (row) => { + row.label = `number-pending` + }) + finish = async () => { + children.utils.resolveSync() + await transaction.when(`settled`) + finish = async () => {} + } + check() + cut = `settled` + const pending = expected() + facts = facts.map((fact) => + fact.row.id === 1 + ? { ...fact, row: { ...initial[0]! }, pending: false } + : fact, + ) + allowed = [pending, expected()] + await finish() + check() + cut = `remote-next-use` + const settled = expected() + facts = facts.map((fact) => + fact.row.id === `1` + ? { ...fact, row: { ...fact.row, label: `string-remote` } } + : fact, + ) + allowed = [settled, expected()] + children.utils.begin() + children.utils.write({ type: `update`, value: { ...facts[1]!.row } }) + children.utils.commit() + check() + if (report.violations.length) + throw new MetadataCallbackOracleFailure(report) + }, [ + () => finish(), + () => live.cleanup(), + () => parents.cleanup(), + () => children.cleanup(), + ]) + return report +} diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts new file mode 100644 index 0000000000..c1d38e6a75 --- /dev/null +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts @@ -0,0 +1,346 @@ +/** + * # Reserved fields must be correct before user JavaScript consumes them. + * + * The shared input oracle states the maintainer's source/projection/inline law. + * This finite receiver exhausts 231 root histories, 231 wrapped child histories + * and 165 direct child histories. Another 99 inline-value/rejection histories + * and three direct-source controls exercise the neighboring boundaries. + * It invokes real query callbacks and compares their copied input and ordinary derived output at + * initial readiness, held mutation, settlement and later remote use. The + * materializer receiver uses the same judgment, not a competing child model. + * + * Reconstruction retains every single-field collision and both populated union + * branches with numeric/string keys. Removing path, shape, collision, boolean + * or checkpoint loses a distinct sanitizer shortcut; the calibrated attacks + * below exhibit those consequences. Legal direct source returns cannot carry + * authored overrides. Compiled Collection-valued inputs are the nearby rejected + * form. Recursion is bounded at two QueryRefs, one LEFT join and flat child + * values. This establishes no native/framework or provider handoff. + * + * There is no random campaign or seed replay for this exhaustive finite grammar. + * Reach explicitly ignores unrelated seed environment variables. A calibration- + * only selection enters no receiving campaign; a partial receiving selection + * still owes the selected campaign's entire required matrix. + */ +import { afterAll, expect, test } from 'vitest' +import { declareOracle } from '../oracle-declaration.js' +import { createOracleReach } from '../oracle-reach.js' +import { + expectedMetadataInput, + expectedMetadataRecord, + expectedSourceVirtualFields, + metadataCallbackHistories, + metadataCallbackReceiverHistories, + metadataInlineHistories, + metadataInputViolations, + metadataOutputViolations, +} from './virtual-row-fields-input-oracle.js' +import { + captureMetadataInput, + runInlineMetadataCallbackHistory, + runMetadataCallbackHistory, +} from './virtual-row-fields-input-driver.js' +import type { MetadataSourceFact } from './virtual-row-fields-input-oracle.js' + +const cuts = [`initial`, `held`, `settled`, `remote-next-use`] as const +const key = (history: { + shape: string + collision: string + authoredPending: boolean +}) => `${history.shape}/${history.collision}/${history.authoredPending}` +// Vitest interprets dollar-prefixed words in table titles as placeholders. +// Escape only the display label; the grammar and reach retain exact names. +const callbackCases = metadataCallbackReceiverHistories.map((history) => ({ + ...history, + label: `${history.receiver}/${history.path}/${key(history)}`.replaceAll( + `$`, + `field:`, + ), +})) +const inlineCases = metadataInlineHistories.map((history) => ({ + ...history, + label: `${history.form}/${key(history)}`.replaceAll(`$`, `field:`), +})) +const callbackReach = createOracleReach( + `functional metadata callback`, + [ + ...metadataCallbackReceiverHistories.flatMap((history) => + cuts.map( + (cut) => `${history.receiver}/${history.path}/${key(history)}/${cut}`, + ), + ), + ...[`root`, `inline-child`, `inline-child-direct`].flatMap((receiver) => + cuts.map((cut) => `${receiver}/direct-source/${cut}`), + ), + ], + { replaying: false }, +) +const inlineReach = createOracleReach( + `inline metadata callback`, + metadataInlineHistories.flatMap((history) => + history.form === `collection` + ? [`${history.form}/${key(history)}/rejected`] + : cuts.map((cut) => `${history.form}/${key(history)}/${cut}`), + ), + { replaying: false }, +) +let callbackEntered = false +let inlineEntered = false +afterAll(() => { + if (callbackEntered) callbackReach.assertReached() + if (inlineEntered) inlineReach.assertReached() + if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { + for (const [name, reach] of [ + [`callback`, callbackReach], + [`inline`, inlineReach], + ] as const) { + const counts = [...reach.counts().values()] + console.info(`${name} metadata reach`, { + states: counts.length, + reached: counts.filter(Boolean).length, + minimum: Math.min(...counts), + }) + } + } +}) +declareOracle({ + check: [metadataInputViolations, metadataOutputViolations], + model: [expectedMetadataInput], + grammar: [ + metadataCallbackHistories, + metadataCallbackReceiverHistories, + metadataInlineHistories, + ], +}) + +test.each(callbackCases)( + `receives callback metadata before deriving ordinary values: $label`, + async (history) => { + callbackEntered = true + callbackReach.beginHistory() + const report = await runMetadataCallbackHistory( + history, + history.path, + (cut) => + callbackReach.reach( + `${history.receiver}/${history.path}/${key(history)}/${cut}`, + ), + history.receiver, + ) + expect( + report.violations, + JSON.stringify(report.violations[0]), + ).toStrictEqual([]) + }, +) +test.each([`root`, `inline-child`, `inline-child-direct`] as const)( + `receives direct source metadata before deriving ordinary values: %s`, + async (receiver) => { + callbackEntered = true + callbackReach.beginHistory() + const report = await runMetadataCallbackHistory( + { + shape: `functional-direct`, + collision: `none`, + authoredPending: false, + }, + `direct-source`, + (cut) => callbackReach.reach(`${receiver}/direct-source/${cut}`), + receiver, + ) + expect(report.violations).toStrictEqual([]) + }, +) +test.each(inlineCases)( + `receives inline authored metadata or rejects compiled Collection input: $label`, + async (history) => { + inlineEntered = true + inlineReach.beginHistory() + const report = await runInlineMetadataCallbackHistory( + history, + history.form, + (cut) => inlineReach.reach(`${history.form}/${key(history)}/${cut}`), + ) + expect( + report.violations, + JSON.stringify(report.violations[0]), + ).toStrictEqual([]) + }, +) + +const sourceFact: MetadataSourceFact = { + row: { id: 1, parentId: `group`, label: `before`, optional: undefined }, + sourceId: `genuine-source`, + pending: false, +} +// Controls alter observations, never source files. The allowed answer comes +// from the same judgment used for every real receiving history above. +test.each([ + `source-spoof`, + `source-stripped`, + `projection-leak`, + `inline-stripped`, + `own-undefined-lost`, + `mixed-before-after`, + `late-capture-rewrite`, +] as const)(`rejects a hostile callback-input observation: %s`, (fault) => { + const description = { + shape: `structured-spread`, + collision: `all`, + authoredPending: true, + } as const + const projected = { ...description, shape: `functional-spread` } as const + const use = fault === `projection-leak` ? projected : description + const inline = fault === `inline-stripped` + const expected = expectedMetadataInput(sourceFact, use, inline) + const hostile = structuredClone(expected) + if (fault === `source-spoof`) hostile.fields.$collectionId!.value = `spoof` + if (fault === `source-stripped` || fault === `inline-stripped`) + hostile.fields.$key = { present: false, own: false, value: undefined } + if (fault === `projection-leak`) + hostile.fields.$key = { present: true, own: true, value: `spoof` } + if (fault === `own-undefined-lost`) delete hostile.ordinary.optional + const next = expectedMetadataInput( + { + ...sourceFact, + row: { ...sourceFact.row, label: `pending` }, + pending: true, + }, + use, + inline, + ) + if (fault === `mixed-before-after`) hostile.fields = next.fields + if (fault === `late-capture-rewrite`) hostile.ordinary.label = `later` + expect(metadataInputViolations(expected, [expected])).toStrictEqual([]) + expect(metadataInputViolations(hostile, [expected, next])).not.toStrictEqual( + [], + ) +}) +test.each([ + `omitted-peer`, + `duplicate-peer`, + `wrong-key-kind`, + `publication-only-sanitizer`, + `missing-derived-field`, +] as const)(`rejects a hostile derived ordinary output: %s`, (fault) => { + const description = { + shape: `functional-projection`, + collision: `$collectionId`, + authoredPending: false, + } as const + const expected = [ + sourceFact, + { ...sourceFact, row: { ...sourceFact.row, id: `1` } }, + ].map((fact) => ({ + id: fact.row.id, + observed: expectedMetadataInput(fact, description), + })) + const hostile = structuredClone(expected) + if (fault === `omitted-peer`) hostile.pop() + if (fault === `duplicate-peer`) hostile.push(structuredClone(hostile[1]!)) + if (fault === `wrong-key-kind`) hostile[1]!.id = 1 + if (fault === `publication-only-sanitizer`) + hostile[0]!.observed.fields.$collectionId = { + present: true, + own: true, + value: `spoof`, + } + if (fault === `missing-derived-field`) + Reflect.deleteProperty(hostile[0]!, `observed`) + expect(metadataOutputViolations(expected, expected)).toStrictEqual([]) + expect(metadataOutputViolations(hostile, expected)).not.toStrictEqual([]) +}) +test(`copies callback input before later source objects change`, () => { + const description = { + shape: `functional-direct`, + collision: `none`, + authoredPending: false, + } as const + const value = { + ...sourceFact.row, + ...expectedSourceVirtualFields(sourceFact), + } + const capture = captureMetadataInput(value) + value.label = `later` + expect( + metadataInputViolations(capture, [ + expectedMetadataInput(sourceFact, description), + ]), + ).toStrictEqual([]) +}) +test(`accepts readable inherited source metadata at callback input`, () => { + const description = { + shape: `functional-direct`, + collision: `none`, + authoredPending: false, + } as const + const value: object = Object.assign( + Object.create(expectedSourceVirtualFields(sourceFact)), + sourceFact.row, + ) + const capture = captureMetadataInput(value) + const expected = expectedMetadataInput(sourceFact, description) + expect(metadataInputViolations(capture, [expected])).toStrictEqual([]) + expect( + metadataOutputViolations( + [{ id: 1, observed: capture }], + [{ id: 1, observed: expected }], + ), + ).toStrictEqual([]) +}) +test(`copies nested callback values before later child objects change`, () => { + const child = { + ...sourceFact.row, + ...expectedSourceVirtualFields(sourceFact), + } + const capture = captureMetadataInput({ id: `parent`, children: [child] }) + const expected = expectedMetadataRecord({ + id: `parent`, + children: [ + { ...sourceFact.row, ...expectedSourceVirtualFields(sourceFact) }, + ], + }) + child.label = `later` + expect(metadataInputViolations(capture, [expected])).toStrictEqual([]) +}) +test.each([`selected-virtual`, `own-undefined`] as const)( + `rejects an inherited replacement for an authored inline field: %s`, + (fault) => { + const description = { + shape: `functional-projection`, + collision: `$collectionId`, + authoredPending: false, + } as const + const selected = expectedMetadataInput(sourceFact, description, true) + const child: Record = { + ...selected.ordinary, + ...Object.fromEntries( + Object.entries(selected.fields) + .filter(([, field]) => field.present) + .map(([name, field]) => [name, field.value]), + ), + } + const expected = expectedMetadataRecord({ id: `parent`, children: [child] }) + const field = fault === `selected-virtual` ? `$collectionId` : `optional` + const inherited: object = Object.assign( + Object.create({ [field]: child[field] }), + Object.fromEntries( + Object.entries(child).filter(([name]) => name !== field), + ), + ) + const actual = captureMetadataInput({ id: `parent`, children: [inherited] }) + expect( + metadataInputViolations( + captureMetadataInput({ id: `parent`, children: [child] }), + [expected], + ), + ).toStrictEqual([]) + expect(metadataInputViolations(actual, [expected])).not.toStrictEqual([]) + expect( + metadataOutputViolations( + [{ id: `parent`, observed: actual }], + [{ id: `parent`, observed: expected }], + ), + ).not.toStrictEqual([]) + }, +) diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.ts new file mode 100644 index 0000000000..aaab7f4b9b --- /dev/null +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.ts @@ -0,0 +1,267 @@ +/** + * # What may user JavaScript learn from a query input? + * + * Authority: the maintainer's O12 decision, recorded in the RC challenge on + * 2026-10-28, and ARCHITECTURE.md's functional-input boundary. Direct source + * returns and structured source spreads expose genuine source metadata. + * Own-shape projections (including functional spreads) expose no reserved + * fields before a downstream callback. Inline include values keep authored + * shape. Collection-valued compiled inputs remain inadmissible. + * + * This small model uses authored source facts and syntax descriptions, not + * compiler provenance or observed values. Callback input is a model observation + * of one runtime value; it is not a published Collection row. An ordinary + * derived output carries this observation so publication-only sanitization + * cannot erase an earlier violation. The finite grammar bounds recursion at + * two QueryRefs and joins at one LEFT join. Provider/framework timing, arbitrary + * objects/descriptors and same-key overlapping transactions are outside it. + */ +import { isDeepStrictEqual } from 'node:util' +import { declareOracle } from '../oracle-declaration.js' +import { virtualFieldNames } from './virtual-row-fields-support.js' +import { expectedSourceMetadata } from './virtual-row-fields-source-oracle.js' +import { + metadataCollisionCases, + metadataProjectionShapes, +} from './virtual-row-fields-grammar-oracle.js' + +export type MetadataInputShape = (typeof metadataProjectionShapes)[number] +export type MetadataInputRow = { + id: string | number + parentId: string + label: string + // An explicit undefined ordinary property must survive both boundaries. + optional: undefined +} +export type MetadataSourceFact = { + row: MetadataInputRow + sourceId: string + pending: boolean +} +export type MetadataCollision = (typeof metadataCollisionCases)[number] +export type MetadataInputDescription = MetadataCollision & { + shape: MetadataInputShape +} +export const metadataInputDescriptions = metadataProjectionShapes.flatMap( + (shape) => + metadataCollisionCases + .filter( + ({ collision }) => + shape !== `functional-direct` || collision === `none`, + ) + .map((collision) => ({ shape, ...collision })), +) +export const metadataCallbackPaths = [ + `query-ref`, + `nested-query-ref`, + `joined-from`, + `joined-source`, + `union-first`, + `union-last`, + `union-query-ref`, +] as const +export type MetadataCallbackPath = (typeof metadataCallbackPaths)[number] +export const metadataCallbackHistories = metadataInputDescriptions.flatMap( + (description) => + metadataCallbackPaths.map((path) => ({ ...description, path })), +) +export const metadataCallbackReceiverHistories = + metadataCallbackHistories.flatMap((history) => + ([`root`, `inline-child`, `inline-child-direct`] as const) + // A flat union has no named source reference for include correlation. + // Admission requires the QueryRef used by union-query-ref or the wrapped + // receiver; direct union-first/last are not legal include children. + .filter( + (receiver) => + receiver !== `inline-child-direct` || + (history.path !== `union-first` && history.path !== `union-last`), + ) + .map((receiver) => ({ ...history, receiver })), + ) +export const metadataInlineHistories = metadataInputDescriptions.flatMap( + (description) => + ([`array`, `materialized`, `collection`] as const).map((form) => ({ + ...description, + form, + })), +) +export type MetadataInputSnapshot = { + present: boolean + ordinary: Record + fields: Record +} + +// This is the existing source-row law, shared with root publication. A pending +// unconfirmed update drops at settlement; a later remote write stays remote. +export function expectedSourceVirtualFields(fact: { + row: { id: string | number } + sourceId: string + pending: boolean +}) { + return expectedSourceMetadata(fact.row.id, fact.sourceId, fact.pending) +} +export function authoredMetadataCollision( + description: MetadataCollision, +): Record { + return Object.fromEntries( + virtualFieldNames + .filter( + (field) => + description.collision === field || description.collision === `all`, + ) + .map((field) => [ + field, + field === `$hasPendingWrites` + ? description.authoredPending + : field === `$origin` + ? `local` + : `spoof-${field}`, + ]), + ) +} +function ordinaryProjection( + fact: MetadataSourceFact, + shape: MetadataInputShape, +): Record { + if (shape === `structured-projection`) + return { + id: fact.row.id, + label: fact.row.label, + hasPendingWrites: fact.pending, + } + if (shape === `functional-projection`) + return { + id: fact.row.id, + label: fact.row.label, + optional: undefined, + } + return { ...fact.row } +} +export function expectedMetadataInput( + fact: MetadataSourceFact, + description: MetadataInputDescription, + inline = false, +): MetadataInputSnapshot { + const ordinary = ordinaryProjection(fact, description.shape) + const sourceOwned = + description.shape === `structured-spread` || + description.shape === `functional-direct` + const fields = inline + ? { + ...(description.shape.endsWith(`projection`) + ? {} + : expectedSourceVirtualFields(fact)), + ...authoredMetadataCollision(description), + } + : sourceOwned + ? expectedSourceVirtualFields(fact) + : {} + return expectedMetadataRecord({ ...ordinary, ...fields }) +} +// A record snapshot is also the common judgment for inline parents. It keeps +// missing fields distinct from own undefined and checks every ordinary field. +export function expectedMetadataRecord( + record: Record | undefined, +): MetadataInputSnapshot { + return { + present: record !== undefined, + ordinary: Object.fromEntries( + Object.entries(record ?? {}).filter( + ([key]) => !virtualFieldNames.some((field) => field === key), + ), + ), + fields: Object.fromEntries( + virtualFieldNames.map((field) => [ + field, + { + present: record !== undefined && field in record, + own: record !== undefined && Object.hasOwn(record, field), + value: record?.[field], + }, + ]), + ), + } +} +// Reserved metadata promises presence and readable value, not descriptor +// layout. Keep `own` in captures as diagnostics without requiring it for a +// present field. An absent field cannot legally be own. This applies only to +// top-level callback metadata. Nested inline selected records stay in ordinary +// data and retain their authored own fields, including own undefined. +function readableMetadataInput(snapshot: MetadataInputSnapshot) { + return { + ...snapshot, + fields: Object.fromEntries( + Object.entries(snapshot.fields).map(([name, field]) => [ + name, + field.present ? { present: true, value: field.value } : field, + ]), + ), + } +} +export function metadataInputViolations( + actual: MetadataInputSnapshot, + allowed: ReadonlyArray, +): Array { + return allowed.some((expected) => + isDeepStrictEqual( + readableMetadataInput(actual), + readableMetadataInput(expected), + ), + ) + ? [] + : [`callback input differs from every permitted authored source state`] +} +export function metadataOutputViolations( + actual: ReadonlyArray<{ + id: string | number + observed: MetadataInputSnapshot + }>, + expected: ReadonlyArray<{ + id: string | number + observed: MetadataInputSnapshot + }>, +): Array { + const byId = (a: { id: string | number }, b: { id: string | number }) => + `${typeof a.id}:${a.id}`.localeCompare(`${typeof b.id}:${b.id}`) + // The derived output carries the recorder's diagnostic ownership flag too. + // Judge its metadata facts by the same readable-value law, while exact outer + // fields, typed keys, ordinary input fields and occurrence counts stay strict. + const stripDiagnostics = (rows: ReadonlyArray) => + rows.map((value) => { + if (!value || typeof value !== `object`) return value + const row = value as Record + const observed = row.observed + if ( + !observed || + typeof observed !== `object` || + !(`fields` in observed) || + !observed.fields + ) + return row + return { + ...row, + observed: readableMetadataInput(observed as MetadataInputSnapshot), + } + }) + return isDeepStrictEqual( + stripDiagnostics([...actual].sort(byId)), + stripDiagnostics([...expected].sort(byId)), + ) + ? [] + : [`derived ordinary output differs from the callback-input law`] +} + +declareOracle({ + model: [ + expectedSourceVirtualFields, + expectedMetadataInput, + expectedMetadataRecord, + ], + check: [metadataInputViolations, metadataOutputViolations], + grammar: [ + metadataInputDescriptions, + metadataCallbackHistories, + metadataCallbackReceiverHistories, + metadataInlineHistories, + ], +}) From baa273760e790b57880eb39d35283b23c5b3bbda Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 10:16:51 -0600 Subject: [PATCH 02/19] chore: add callback metadata fix changeset --- .changeset/fix-correlated-callback-metadata.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fix-correlated-callback-metadata.md diff --git a/.changeset/fix-correlated-callback-metadata.md b/.changeset/fix-correlated-callback-metadata.md new file mode 100644 index 0000000000..66a768c9d1 --- /dev/null +++ b/.changeset/fix-correlated-callback-metadata.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Fix reserved metadata in functional callback inputs inside correlated includes. Preserve genuine source metadata and authored inline values. From a30256cfbaf99138f274db224362325c37ea1da4 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 10:26:17 -0600 Subject: [PATCH 03/19] Preserve inline metadata while sanitizing callback sources --- docs/contributing/oracle-coverage.md | 18 +-- .../2026-10-29-pr-2171-deferred-oracles.md | 5 +- .../oracle-reviews/callback-metadata.md | 108 +++++++++++------- packages/db/src/query/compiler/index.ts | 19 ++- packages/db/src/query/live/ARCHITECTURE.md | 7 +- .../query/virtual-row-fields-input-driver.ts | 89 +++++++++++---- .../virtual-row-fields-input-oracle.test.ts | 19 ++- .../query/virtual-row-fields-input-oracle.ts | 11 +- 8 files changed, 185 insertions(+), 91 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 69cb4470e8..b226faf303 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -809,18 +809,22 @@ readiness, held mutation, settlement and remote next use. Complete before/after source facts permit retraction evaluation. Startup LEFT joins may expose an absent side; established outputs require both numeric/string-key rows. Reserved metadata is judged by presence/readable value, with ownership diagnostic. -Ordinary own undefined stays distinct from absence. Another 66 histories retain -nested inline array/materialized authored values, and 33 reject compiled -Collection inputs before callbacks. All 2,520 callback and 297 inline/rejection +Ordinary own undefined stays distinct from absence. Another 132 histories retain +nested inline array/materialized authored values through direct and QueryRef +paths, and 66 reject compiled Collection inputs before callbacks. All 2,520 +callback and 594 inline/rejection states reach once; 17 controls calibrate the same judgments. Partial receiving selections still owe the selected campaign's full reach matrix. The restored owner reproduced 203 correlated-child REDs on unchanged main. Adding the direct receiving paths before production repair raised that to 348 -REDs with 398 passes. The compiler now enforces source/projection metadata at -QueryRef and joined-source boundaries independently of final inline output. -All 746 owner tests pass. A QueryRef-only mutant leaves 29 joined-source -violations; sanitizing every inline child breaks 58 authored-value histories. +REDs with 398 passes. The expanded 845-test owner also rejects the first repair in 58 inline QueryRef +histories before the corrected production edit. The compiler now requests +source/projection metadata at FROM and joined-source boundaries when a functional +callback consumes the rows. Inline-only QueryRefs preserve authored shape. +All 845 owner tests pass. A QueryRef-only mutant leaves 58 joined-source +violations; sanitizing every inline child breaks 116 authored-value histories. +Restoring unconditional source sanitization breaks 58 inline QueryRef histories. The [runtime repair record](oracle-reviews/callback-metadata.md) retains the commands, distinguishing traces, source fingerprints and limits. Historical instrument acceptance remains in diff --git a/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md b/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md index 21fb8c6fb0..2bfdaa5124 100644 --- a/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md +++ b/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md @@ -20,8 +20,9 @@ No follow-up PR is claimed until one is actually created. owner has 377 passes / 203 intended REDs and all 2,153 cells; the bounded concrete lane has the same seven failures and one compatible pass on main. The runtime follow-up restores this owner and adds direct child receivers. - It reproduces 348 REDs before repair and passes all 746 tests afterward, with - all 2,817 states reached. See [the runtime repair](callback-metadata.md). + It reproduces 348 REDs before repair and passes all 845 tests afterward, with + all 3,114 states reached. Inline QueryRef histories also reject an overbroad + first repair before the corrected production edit. See [the runtime repair](callback-metadata.md). This does not close the separate static metadata receiving-type failures below. - P11/P12: raw undefined becomes null and selected empty records disappear. Their root/QueryRef runtime owner retains 18 REDs, correlated receiver 72 REDs, diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 9524882cf2..f887085eac 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -29,8 +29,11 @@ paths run at root and through a wrapped correlated child: 231 histories each. Five paths also run directly as the correlated child: 165 histories. Flat union children have no named source reference for include correlation, so their supported forms use the wrapper or union-query-ref path. They remain covered. -Three direct-source controls, 66 inline array/materialized histories, 33 compiled -Collection rejections and 17 checker/recorder controls complete 746 tests. +Three direct-source controls, 132 inline array/materialized histories, 66 compiled +Collection rejections and 17 checker/recorder controls complete 845 tests. Inline +values cross direct and QueryRef paths. The QueryRef path correlates each parent +with one typed-key child, so it preserves authored child fields without a child +functional callback. Every receiving history observes initial readiness, a held optimistic update, settlement without confirmation, and a later remote update. Source rows have @@ -80,42 +83,59 @@ No comparison, reach minimum or run budget was weakened to obtain GREEN. ## Repair and discriminators -A recursive QueryRef or joined subquery exports a source value even when its -consumer is inside an inline include. Those source boundaries now use the -existing source/projection metadata mode. The enclosing include's final value -keeps the existing inline mode. The obsolete `processFrom` mode parameter is -removed. No new metadata registry, state, traversal or public API is added. - -All 746 tests pass with all 2,817 reach states at count one. Seven neighboring -suites pass 1,883 tests: root metadata, functional projection/input rejection, -route-context transport, public container copying, include query shapes and -subquery user values. Focused no-emit TypeScript, ESLint and changed-file oracle -lint pass. After integrating #2301 (`bfd1da1a7e752d66fd667adfd3cdf54f60732b22`), -the normal package pnpm configuration also passes all 746 tests and all reach. -No local build or dependency installation ran. - -Two deliberately wrong repairs ran in copied source trees, without changing the -reviewed worktree: +The first repair enabled source/projection metadata mode at every recursive +QueryRef and joined source. A neighboring-path probe found that this was too +broad: an inline child with a QueryRef, but no functional callback, lost its +authored reserved fields. The existing inline grammar used direct children only. + +Before another production edit, the inline grammar added QueryRef consumption +across all shapes, collisions and materialization forms: 99 more histories. +The first repair then failed 58 shared callback/output comparisons and passed +787 tests. For example, `query-ref/array/structured-projection/$key/false` +lost the child's authored `$key: "spoof-$key"` at the initial callback capture. +The unchanged baseline compiler passes all these inline histories and still +fails the original 348 callback comparisons: 497 passes out of 845 tests. +These runs distinguish both accepted laws before the corrected repair. + +The corrected repair requests source/projection metadata mode when the local +query has a functional selection, or when the enclosing query already requires +Collection publication mode. It passes that mode into FROM and joined-source +compilation. An inline-only QueryRef retains its authored selected shape. +Existing mode parameters and propagation remain. The production diff changes +two call sites. It adds no metadata registry, state, traversal or public API. + +All 845 tests pass. Every one of the 2,520 callback and 594 inline/rejection +states reaches once. Seven neighboring suites pass 1,883 tests: root metadata, +functional projection/input rejection, route-context transport, public container +copying, include query shapes and subquery user values. Focused no-emit TypeScript, +ESLint and changed-file oracle lint pass. The normal package pnpm configuration +also passes all 845 owner tests after integrating #2301 +(`bfd1da1a7e752d66fd667adfd3cdf54f60732b22`). No local build or installation ran. + +Three wrong repairs ran sequentially in copied source trees. They did not change +the reviewed worktree: | Mutant | Intended wrong design | Result at the shared comparison | | --- | --- | --- | -| QueryRef-only | Keep the old joined-source mode | 29 failures, 717 passes | -| Sanitize inline | Sanitize every final child instead of preserving authored inline values | 58 failures, 688 passes | +| QueryRef-only | Keep the old joined-source mode | 58 failures, 787 passes | +| Sanitize inline | Sanitize every final child instead of preserving authored inline values | 116 failures, 729 passes | +| Unconditional sources | First repair, ignoring whether a callback consumes the rows | 58 failures, 787 passes | -Both mutants reached the intended callback/output comparisons. They are assertion -failures, not timeouts or setup errors. The original 348-failure RED is the -original-defect discriminator. The direct receiver prevents an outer QueryRef -from concealing an unrepaired joined-source boundary. +Every failure is a `MetadataCallbackOracleFailure` at the intended input/output +comparison. No mutant timed out or failed setup. The original 348-failure RED +rejects the original defect. Direct receivers distinguish an unrepaired joined +source, and the expanded inline paths reject both blanket sanitization repairs. ## Prep review and loss audit -Separate read-only review and simplification passes found no confirmed production -or oracle-enforcement defect. Both found stale opening counts. Those counts are -corrected. The review also required these planned coverage/architecture updates. +The initial read-only review and simplification passes found stale opening +counts and required coverage/architecture updates. They missed the inline QueryRef +regression found by the later neighboring-path probe. That gap required the +grammar extension and corrected repair described above. A proposed shared-query cache conflict was tested through actual Effects and live-query Collections. Each receiver reused one projected query through a -QueryRef, then through both union branch orders. All six probes passed: the +QueryRef, then through both union branch orders. All six probes passed on both repairs: the nested callback saw no `$collectionId`. The direct Effect branch retained its authored reserved output, which is separate from downstream callback input. The initial Collection-union fixture supplied conflicting custom public keys; @@ -143,22 +163,32 @@ The source-level review distinguished final inline/Effect values from callback input. Its finite probes supply useful cross-path evidence. The reviewers did not independently rerun the full owner or mutation campaign. -Four retained items reconcile as two fixed documentation items, one duplicate -count correction and one refuted cache hypothesis. Both review passes are useful -for this bounded patch. Neither supplies a whole-library correctness verdict. +The initial four review items and the new inline finding reconcile as three +fixed items, one duplicate count correction and one refuted cache hypothesis. +The new finding retains both its first-repair RED and corrected GREEN evidence. +Initial reviewers provided useful documentation and cache analysis, but their +missed inline path limits confidence in their completeness. The final independent +review and simplification pass found no new defect or useful code reduction. +The reviewer also ran two inline LEFT-join controls without a child callback: +one selected QueryRef on FROM, the other on the joined side. Both selected the +whole projected child into `toArray()` and retained authored +`$collectionId: "authored"` after preload. These are supplemental initial-cut +controls, not generated lifecycle coverage. Their repeated documentation notes +link to the original count item, giving seven raw items: three fixed, one +refuted and three duplicates. Neither review supplies a whole-library verdict. ## Rubric accounting and limits ORC-001–005: accepted authority, independent source facts, literate layers, finite legal grammar, real callback captures and exact derived output cuts. -Removing receiver, path, collision, pending boolean or checkpoint loses a +Removing receiver, callback path, inline path, collision, pending boolean or checkpoint loses a named source/projection/inline distinction. Both populated union positions and both typed keys remain represented. Direct-source controls distinguish a blanket stripping rule. The rejected flat-union form bounds admission. -ORC-006: 17 hostile observation/recorder controls and both production mutants. +ORC-006: 17 hostile observation/recorder controls and all three production mutants. ORC-007: exhaustive finite enumeration, so random campaigns and seed replay do -not apply. A valid unrelated `utils.1` seed/path run passes all 746 tests with +not apply. A valid unrelated `utils.1` seed/path run passes all 845 tests with all reach intact. A calibration-only selection passes 17 controls without entering a receiver. Selecting only the three direct-source controls fails the missing-matrix reach assertion. An omitted replay property was a configuration @@ -169,7 +199,7 @@ failure reports and failure-preserving cleanup. ORC-011: the concrete shared-cac hypothesis has six same-path probes; no remaining shared-model hypothesis was identified. ORC-012: this versioned record and source fingerprints below. ORC-013: single/all/no collisions, typed peers, source/new/inline forms and the -two mutants distinguish plausible shortcut repairs. ORC-014: the declared +three mutants distinguish plausible shortcut repairs. ORC-014: the declared boundary is core runtime compilation, with no real-provider/framework timing claim. ORC-015: every named callback and inline/rejection state reaches once. ORC-016: all histories and controls use the shared independent judgments. @@ -199,7 +229,7 @@ original failures without temporary log files. | Source | SHA256 | | --- | --- | -| `packages/db/src/query/compiler/index.ts` | `3982fc56163cfc44583a737b9bbf91c5aece9d6c0212324721309646a9eb488e` | -| `packages/db/tests/query/virtual-row-fields-input-oracle.ts` | `99a7357c283731adc7e7552a9e66e4b3d4df0df58576db82f4d9e5f0ef9f9265` | -| `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `31a19f9988ca46b82eba13b434f8d3e1deb5ff972b7fe63df64cbb8898693b79` | -| `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `2d417011231e109a9ed5194f417063bc41ac46cc36ec6405aa61e8b60408cb6b` | +| `packages/db/src/query/compiler/index.ts` | `07c93e178ddf176af57f85b86479a3f2a762fd96b726f31755e7ed13eafc2fa3` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.ts` | `25f729a6444f7e35526a499aefd7c624acba06348d18f4c31b9d3c2f5d7259a6` | +| `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `f1cb1ec6bc441679030cfe08cead25478caf419b49e77a582db6fe74407cf953` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `dd6d015e7bff43e6af6038dc70e257b34cf700748eafbcb30bf7a7ae6182a707` | diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 80547a6f07..0989d40c48 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -509,7 +509,9 @@ function compileQueryInternal( aliasRemapping, sourceWhereClauses, parentKeyStream, - publishCollectionRoot, + // Sanitize source rows before a callback consumes them. Inline values + // without a callback keep their authored selected shape. + publishCollectionRoot || query.fnSelect !== undefined, ) Object.assign(sources, fromSources) const sourceCarriesInternalRouteState = @@ -622,8 +624,12 @@ function compileQueryInternal( ignoreWindow, rawQuery, query.from, - // Joined subqueries are callback inputs even inside inline includes. - (...args) => compileQueryInternal(args, true, true), + (...args) => + compileQueryInternal( + args, + true, + publishCollectionRoot || query.fnSelect !== undefined, + ), aliasToCollectionId, aliasRemapping, sourceWhereClauses, @@ -1647,6 +1653,7 @@ function processFromClause( aliasRemapping, sourceWhereClauses, parentKeyStream, + publishCollectionRoot, ) return { @@ -1693,6 +1700,7 @@ function processFromClause( aliasRemapping, sourceWhereClauses, parentKeyStream, + publishCollectionRoot, ) if (!mainAlias) { @@ -1934,6 +1942,7 @@ function processFrom( aliasRemapping: Record, sourceWhereClauses: Map>, parentKeyStream?: KeyedStream, + publishCollectionRoot = false, ): { alias: string input: KeyedStream @@ -1987,9 +1996,7 @@ function processFrom( undefined, ], true, - // A QueryRef exports a source row, not its enclosing inline value. - // Enforce source/projection metadata before any consumer runs. - true, + publishCollectionRoot, ) // Pull up alias mappings from subquery to parent scope. diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 9d78fdadf3..82b6e0e516 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -396,9 +396,10 @@ authored selected shape. Sanitizing only the eventual Collection publication cannot satisfy this law, because user code may already have derived an ordinary field from a spoofed input. -The compiler enforces this source/projection boundary when a QueryRef or joined -subquery exports its result, including inside a correlated inline child. The -include's final authored value retains its separate inline shape. The runtime +The compiler enforces this source/projection boundary when a functional callback +consumes a QueryRef or joined subquery result, including inside a correlated +inline child. An inline-only QueryRef and the include's final authored value +retain their separate inline shape. The runtime owner checks root, wrapped child and direct child callbacks at initial readiness, held mutation, settlement and remote next use. Its bounded repair evidence is in [the callback metadata record](../../../../../docs/contributing/oracle-reviews/callback-metadata.md). diff --git a/packages/db/tests/query/virtual-row-fields-input-driver.ts b/packages/db/tests/query/virtual-row-fields-input-driver.ts index a6f7b3ab57..e94ca74ffb 100644 --- a/packages/db/tests/query/virtual-row-fields-input-driver.ts +++ b/packages/db/tests/query/virtual-row-fields-input-driver.ts @@ -453,49 +453,84 @@ export async function runInlineMetadataCallbackHistory( description: MetadataInputDescription, form: `array` | `materialized` | `collection`, reached: (cut: CallbackInputCut | `rejected`) => void, + path: `direct` | `query-ref` = `direct`, ): Promise { const initial = freshRows() const children = source(`callback-inline-child`, initial) - const parents = source(`callback-inline-parent`, [ - { id: `parent`, parentId: `group`, label: `parent`, optional: undefined }, - ]) + const parentRows: Array = + path === `direct` + ? [ + { + id: `parent`, + parentId: `group`, + label: `parent`, + optional: undefined, + }, + ] + : initial.map((row) => ({ ...row })) + const parents = source(`callback-inline-parent`, parentRows) let facts = initial.map((row) => ({ row: { ...row }, sourceId: children.id, pending: false, })) let cut: CallbackInputCut = `initial` + // A QueryRef in a child without a functional callback preserves the inline + // selected shape too. One parent per typed key gives its outer correlation + // a named source field without requiring a projected parentId. const expected = () => - expectedMetadataRecord({ - id: `parent`, - children: rowsFromSnapshots( - facts.map((fact) => expectedMetadataInput(fact, description, true)), - ), - }) - let allowed = [expected()] + parentRows.map((parent) => + expectedMetadataRecord({ + id: parent.id, + children: rowsFromSnapshots( + facts + .filter((fact) => path === `direct` || fact.row.id === parent.id) + .map((fact) => expectedMetadataInput(fact, description, true)), + ), + }), + ) + let allowed = expected() // The controlled source publishes its initial child batch atomically. A // startup callback can precede that batch or consume its complete rows. - allowed.push(expectedMetadataRecord({ id: `parent`, children: [] })) + allowed.push( + ...parentRows.map((parent) => + expectedMetadataRecord({ id: parent.id, children: [] }), + ), + ) const calls: Array<{ cut: CallbackInputCut; input: MetadataInputSnapshot }> = [] const build = () => createLiveQueryCollection({ query: (q) => { const included = q.from({ parent: parents }).select(({ parent }) => { - const selected = projectInput( - children, - `child`, - description, - parent.parentId, - ).orderBy(({ $selected }) => $selected.label) + const selected = + path === `direct` + ? projectInput( + children, + `child`, + description, + parent.parentId, + ).orderBy(({ $selected }) => $selected.label) + : new Query() + .from({ + nested: projectInput(children, `child`, description), + }) + .where(({ nested }) => eq(nested.id, parent.id)) + .orderBy(({ nested }) => nested.label) return { id: parent.id, children: form === `collection` - ? include(selected) + ? include( + selected as unknown as QueryBuilder, + ) : form === `array` - ? toArray(selected) - : materialize(selected), + ? toArray( + selected as unknown as QueryBuilder, + ) + : materialize( + selected as unknown as QueryBuilder, + ), } }) return q.from({ nested: included }).fn.select(({ nested }) => { @@ -574,10 +609,14 @@ export async function runInlineMetadataCallbackHistory( expected: allowed, }) const actual = outputRows(live.toArray) - const next = [{ id: `parent`, observed: expected() }] + const next = expected().map((observed, index) => ({ + id: parentRows[index]!.id, + observed, + })) for (const kind of metadataOutputViolations(actual, next)) report.violations.push({ cut, kind, actual, expected: next }) - if (premise && current.length && live.toArray.length === 1) reached(cut) + if (premise && current.length && live.toArray.length === parentRows.length) + reached(cut) } let finish = async () => {} await withOracleCleanup(async () => { @@ -594,7 +633,7 @@ export async function runInlineMetadataCallbackHistory( } : fact, ) - allowed = [before, expected()] + allowed = [...before, ...expected()] const transaction = children.update(1, (row) => { row.label = `number-pending` }) @@ -611,7 +650,7 @@ export async function runInlineMetadataCallbackHistory( ? { ...fact, row: { ...initial[0]! }, pending: false } : fact, ) - allowed = [pending, expected()] + allowed = [...pending, ...expected()] await finish() check() cut = `remote-next-use` @@ -621,7 +660,7 @@ export async function runInlineMetadataCallbackHistory( ? { ...fact, row: { ...fact.row, label: `string-remote` } } : fact, ) - allowed = [settled, expected()] + allowed = [...settled, ...expected()] children.utils.begin() children.utils.write({ type: `update`, value: { ...facts[1]!.row } }) children.utils.commit() diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts index c1d38e6a75..a7ddf66a3d 100644 --- a/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts @@ -3,7 +3,7 @@ * * The shared input oracle states the maintainer's source/projection/inline law. * This finite receiver exhausts 231 root histories, 231 wrapped child histories - * and 165 direct child histories. Another 99 inline-value/rejection histories + * and 165 direct child histories. Another 198 inline-value/rejection histories * and three direct-source controls exercise the neighboring boundaries. * It invokes real query callbacks and compares their copied input and ordinary derived output at * initial readiness, held mutation, settlement and later remote use. The @@ -59,7 +59,10 @@ const callbackCases = metadataCallbackReceiverHistories.map((history) => ({ })) const inlineCases = metadataInlineHistories.map((history) => ({ ...history, - label: `${history.form}/${key(history)}`.replaceAll(`$`, `field:`), + label: `${history.path}/${history.form}/${key(history)}`.replaceAll( + `$`, + `field:`, + ), })) const callbackReach = createOracleReach( `functional metadata callback`, @@ -79,8 +82,10 @@ const inlineReach = createOracleReach( `inline metadata callback`, metadataInlineHistories.flatMap((history) => history.form === `collection` - ? [`${history.form}/${key(history)}/rejected`] - : cuts.map((cut) => `${history.form}/${key(history)}/${cut}`), + ? [`${history.path}/${history.form}/${key(history)}/rejected`] + : cuts.map( + (cut) => `${history.path}/${history.form}/${key(history)}/${cut}`, + ), ), { replaying: false }, ) @@ -159,7 +164,11 @@ test.each(inlineCases)( const report = await runInlineMetadataCallbackHistory( history, history.form, - (cut) => inlineReach.reach(`${history.form}/${key(history)}/${cut}`), + (cut) => + inlineReach.reach( + `${history.path}/${history.form}/${key(history)}/${cut}`, + ), + history.path, ) expect( report.violations, diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.ts index aaab7f4b9b..d764638de6 100644 --- a/packages/db/tests/query/virtual-row-fields-input-oracle.ts +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.ts @@ -80,10 +80,13 @@ export const metadataCallbackReceiverHistories = ) export const metadataInlineHistories = metadataInputDescriptions.flatMap( (description) => - ([`array`, `materialized`, `collection`] as const).map((form) => ({ - ...description, - form, - })), + ([`array`, `materialized`, `collection`] as const).flatMap((form) => + ([`direct`, `query-ref`] as const).map((path) => ({ + ...description, + form, + path, + })), + ), ) export type MetadataInputSnapshot = { present: boolean From 9bd623cd5621a2815c61f4aab206725e30f6ae2e Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 10:44:12 -0600 Subject: [PATCH 04/19] test(db): preserve named array fields in callback recorder --- docs/contributing/oracle-coverage.md | 6 +- .../2026-10-29-pr-2171-deferred-oracles.md | 2 +- .../oracle-reviews/callback-metadata.md | 84 +++++++++++++++++-- .../query/virtual-row-fields-input-driver.ts | 16 +++- .../virtual-row-fields-input-oracle.test.ts | 33 ++++++++ 5 files changed, 130 insertions(+), 11 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index b226faf303..384421996d 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -813,7 +813,7 @@ Ordinary own undefined stays distinct from absence. Another 132 histories retain nested inline array/materialized authored values through direct and QueryRef paths, and 66 reject compiled Collection inputs before callbacks. All 2,520 callback and 594 inline/rejection -states reach once; 17 controls calibrate the same judgments. Partial receiving +states reach once; 19 controls calibrate the same judgments. Partial receiving selections still owe the selected campaign's full reach matrix. The restored owner reproduced 203 correlated-child REDs on unchanged main. @@ -822,7 +822,9 @@ REDs with 398 passes. The expanded 845-test owner also rejects the first repair histories before the corrected production edit. The compiler now requests source/projection metadata at FROM and joined-source boundaries when a functional callback consumes the rows. Inline-only QueryRefs preserve authored shape. -All 845 owner tests pass. A QueryRef-only mutant leaves 58 joined-source +All 847 owner tests pass, including two controls that reject extra named own +fields on inline arrays. The recorder preserves those fields without changing +the model or either judgment. A QueryRef-only mutant leaves 58 joined-source violations; sanitizing every inline child breaks 116 authored-value histories. Restoring unconditional source sanitization breaks 58 inline QueryRef histories. The [runtime repair record](oracle-reviews/callback-metadata.md) retains the diff --git a/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md b/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md index 2bfdaa5124..415dab28a4 100644 --- a/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md +++ b/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md @@ -20,7 +20,7 @@ No follow-up PR is claimed until one is actually created. owner has 377 passes / 203 intended REDs and all 2,153 cells; the bounded concrete lane has the same seven failures and one compatible pass on main. The runtime follow-up restores this owner and adds direct child receivers. - It reproduces 348 REDs before repair and passes all 845 tests afterward, with + It reproduces 348 REDs before repair and passes all 847 tests afterward, with all 3,114 states reached. Inline QueryRef histories also reject an overbroad first repair before the corrected production edit. See [the runtime repair](callback-metadata.md). This does not close the separate static metadata receiving-type failures below. diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index f887085eac..1727504330 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -30,7 +30,7 @@ Five paths also run directly as the correlated child: 165 histories. Flat union children have no named source reference for include correlation, so their supported forms use the wrapper or union-query-ref path. They remain covered. Three direct-source controls, 132 inline array/materialized histories, 66 compiled -Collection rejections and 17 checker/recorder controls complete 845 tests. Inline +Collection rejections and 19 checker/recorder controls complete 847 tests. Inline values cross direct and QueryRef paths. The QueryRef path correlates each parent with one typed-key child, so it preserves authored child fields without a child functional callback. @@ -104,7 +104,7 @@ compilation. An inline-only QueryRef retains its authored selected shape. Existing mode parameters and propagation remain. The production diff changes two call sites. It adds no metadata registry, state, traversal or public API. -All 845 tests pass. Every one of the 2,520 callback and 594 inline/rejection +The pre-review head passed all 845 tests. Every one of the 2,520 callback and 594 inline/rejection states reaches once. Seven neighboring suites pass 1,883 tests: root metadata, functional projection/input rejection, route-context transport, public container copying, include query shapes and subquery user values. Focused no-emit TypeScript, @@ -112,8 +112,9 @@ ESLint and changed-file oracle lint pass. The normal package pnpm configuration also passes all 845 owner tests after integrating #2301 (`bfd1da1a7e752d66fd667adfd3cdf54f60732b22`). No local build or installation ran. -Three wrong repairs ran sequentially in copied source trees. They did not change -the reviewed worktree: +Three wrong repairs ran sequentially in copied source trees before the +independent review. They did not change the reviewed worktree. These counts use +the 845-test owner at `a30256cfb`, before the two added recorder calibrations: | Mutant | Intended wrong design | Result at the shared comparison | | --- | --- | --- | @@ -146,6 +147,13 @@ The probe constructs the following source and nested consumer before selecting QueryRef-only, `unionAll(projected, consumed)`, or its reverse: ```ts +import { createCollection, localOnlyCollectionOptions, Query } from '@tanstack/db' + +const source = createCollection(localOnlyCollectionOptions({ + initialData: [{ id: 1 }], + getKey: (row: { id: number }) => row.id, +})) +const captures: Array<{ present: boolean; value: unknown }> = [] const projected = new Query().from({ source }).select(({ source }) => ({ id: source.id, $collectionId: 'spoof', @@ -177,6 +185,67 @@ controls, not generated lifecycle coverage. Their repeated documentation notes link to the original count item, giving seven raw items: three fixed, one refuted and three duplicates. Neither review supplies a whole-library verdict. +## Independent oracle review after PR creation + +Four subagents used the repository oracle-review skill on +`a30256cfbaf99138f274db224362325c37ea1da4`. They independently challenged laws, +grammar/reach, driver/comparisons, and mutation/process evidence. + +The law review confirmed a historical process gap: the first pre-edit receiving +inventory omitted inline-only QueryRef preservation. That path was added after +the first production repair. The later baseline run cannot satisfy the initial +gate retrospectively. The corrected production edit did have the expanded RED +before its edit. This record preserves that distinction and claims no complete +initial-gate compliance. + +The driver review found a recorder blind spot. `Array.map` copied inline array +entries but dropped named own fields. An ordinary `Object.assign(children, +{ unexpected: "spoof" })`, or its `$collectionId` counterpart, remained publicly +readable but disappeared from capture. Both shared judgments accepted it. +This was a confirmed test-integrity defect, not a reproduced production defect. + +Two new calibration cases ran before the recorder edit. Both failed because +both judgments returned false for rejection. Capture now preserves named own +string fields recursively as well as array entries. `Object.defineProperty` +avoids setter traps. The expected model and both judgments are unchanged. +All 847 owner tests pass: the existing 845 plus two recorder calibrations. +All 2,520 callback and 594 inline/rejection states still reach once. Restoring +the old recorder is the demonstrated wrong design: both new calibrations fail. +The reviewer inspected the fix and found that it closes the reported gap. + +The mutant reviewer independently omitted only the FROM callback-mode change, +leaving joined-source handling intact. On copied sources at the reviewed head, +290 comparisons fail and 555 tests pass. The first distinguishing history is +`inline-child/query-ref/structured-projection/$key/false`, initial: the callback +reads authored `$key: "spoof-$key"` where the model requires absence. Every +failure is `MetadataCallbackOracleFailure`; an earlier configuration EPERM ran +no tests and supplies no evidence. This supplements the three mutants above. + +The grammar review independently reconciled all 845 original tests and the +per-receiver reach matrix. Flattened child output does not prove parent-child +route placement. It retains every child's metadata capture, typed key and +occurrence count; those reviews found no hidden metadata violation from that +flattening. Parent routing remains owned by the separate include oracles. + +The review accounts for two findings: one repaired recorder gap and one +historical gate miss preserved above. No current production defect emerged. +The source-only law and grammar reviews supplied useful independent reasoning. +The driver review supplied an executable false-green witness and a reviewed +repair. The mutant review supplied an additional independent discriminator. +These results support the finite runtime boundary, not a whole-library verdict. + +CI also rejected this record's query example because its source was untyped. +The example now declares its typed source and capture array. Its focused +snippet typecheck passes without a baseline exclusion. That mechanical docs +repair is separate from the two oracle-review findings. + +The full local snippet scan reports 12 failures in unchanged examples and one +stale baseline entry under the reused dependency tree. The earlier CI scan +reported only this record's example as a new failure. No dependency install or +baseline exclusion was used to mask that local environment difference. Full +final-head snippet verification remains for CI; the changed example passes its +focused local check. + ## Rubric accounting and limits ORC-001–005: accepted authority, independent source facts, literate layers, @@ -186,7 +255,8 @@ named source/projection/inline distinction. Both populated union positions and both typed keys remain represented. Direct-source controls distinguish a blanket stripping rule. The rejected flat-union form bounds admission. -ORC-006: 17 hostile observation/recorder controls and all three production mutants. +ORC-006: 19 hostile observation/recorder controls, the three recorded production +mutants and the independent FROM-only mutant. ORC-007: exhaustive finite enumeration, so random campaigns and seed replay do not apply. A valid unrelated `utils.1` seed/path run passes all 845 tests with all reach intact. A calibration-only selection passes 17 controls without @@ -231,5 +301,5 @@ original failures without temporary log files. | --- | --- | | `packages/db/src/query/compiler/index.ts` | `07c93e178ddf176af57f85b86479a3f2a762fd96b726f31755e7ed13eafc2fa3` | | `packages/db/tests/query/virtual-row-fields-input-oracle.ts` | `25f729a6444f7e35526a499aefd7c624acba06348d18f4c31b9d3c2f5d7259a6` | -| `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `f1cb1ec6bc441679030cfe08cead25478caf419b49e77a582db6fe74407cf953` | -| `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `dd6d015e7bff43e6af6038dc70e257b34cf700748eafbcb30bf7a7ae6182a707` | +| `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `c84cf4330e17590b3f9fc55bb449b489a3160e975966823f9ebb4fb4675f18ff` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `948e84d4f3400ebf8b4e867252580a613dc98e3ebb7eef0811e95727f3624337` | diff --git a/packages/db/tests/query/virtual-row-fields-input-driver.ts b/packages/db/tests/query/virtual-row-fields-input-driver.ts index e94ca74ffb..8fa4acd70e 100644 --- a/packages/db/tests/query/virtual-row-fields-input-driver.ts +++ b/packages/db/tests/query/virtual-row-fields-input-driver.ts @@ -75,7 +75,21 @@ class MetadataCallbackOracleFailure extends Error { } function copyData(value: unknown): unknown { - if (Array.isArray(value)) return value.map(copyData) + if (Array.isArray(value)) { + const copy = value.map(copyData) + // Array.map retains indices and holes but omits named own fields. Keep + // those fields too, so capture cannot erase a wrong inline array shape. + for (const key of Object.getOwnPropertyNames(value)) { + if (Object.hasOwn(copy, key)) continue + Object.defineProperty(copy, key, { + value: copyData(Reflect.get(value, key)), + enumerable: true, + configurable: true, + writable: true, + }) + } + return copy + } if (value && typeof value === `object`) return Object.fromEntries( Object.getOwnPropertyNames(value).map((key) => [ diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts index a7ddf66a3d..6bf4115875 100644 --- a/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts @@ -353,3 +353,36 @@ test.each([`selected-virtual`, `own-undefined`] as const)( ).not.toStrictEqual([]) }, ) + +// The inline array is public data too. An extra ordinary or reserved own field +// must survive capture so both judgments can reject the unauthored shape. +test.each([`unexpected`, `$collectionId`] as const)( + `rejects an extra own field on an inline array: %s`, + (field) => { + const child = expectedMetadataInput( + sourceFact, + { + shape: `functional-projection`, + collision: `none`, + authoredPending: false, + }, + true, + ).ordinary + const authored = { id: `parent`, children: [child] } + const expected = expectedMetadataRecord(authored) + const value = structuredClone(authored) + const lawful = captureMetadataInput(value) + expect(metadataInputViolations(lawful, [expected])).toStrictEqual([]) + Object.assign(value.children, { [field]: `spoof` }) + const hostile = captureMetadataInput(value) + const rejected = { + input: metadataInputViolations(hostile, [expected]).length > 0, + output: + metadataOutputViolations( + [{ id: `parent`, observed: hostile }], + [{ id: `parent`, observed: expected }], + ).length > 0, + } + expect(rejected).toStrictEqual({ input: true, output: true }) + }, +) From 5ba869f0fc8398243a45a923828b717f0050f40a Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 11:22:43 -0600 Subject: [PATCH 05/19] fix(db): separate compilation cache publication modes --- .../fix-correlated-callback-metadata.md | 2 +- docs/contributing/oracle-coverage.md | 17 +- .../2026-10-29-pr-2171-deferred-oracles.md | 6 +- .../oracle-reviews/callback-metadata.md | 103 ++++++++-- packages/db/src/query/compiler/index.ts | 16 +- packages/db/src/query/compiler/types.ts | 2 +- packages/db/src/query/live/ARCHITECTURE.md | 4 + .../query/virtual-row-fields-input-driver.ts | 184 +++++++++++++++++- .../virtual-row-fields-input-oracle.test.ts | 54 +++++ .../query/virtual-row-fields-input-oracle.ts | 15 ++ 10 files changed, 375 insertions(+), 28 deletions(-) diff --git a/.changeset/fix-correlated-callback-metadata.md b/.changeset/fix-correlated-callback-metadata.md index 66a768c9d1..be9357ba7c 100644 --- a/.changeset/fix-correlated-callback-metadata.md +++ b/.changeset/fix-correlated-callback-metadata.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Fix reserved metadata in functional callback inputs inside correlated includes. Preserve genuine source metadata and authored inline values. +Fix reserved metadata in functional callback inputs inside correlated includes. Preserve genuine source metadata and authored values when compiled queries share a cache. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 384421996d..6597550cf0 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -816,17 +816,32 @@ callback and 594 inline/rejection states reach once; 19 controls calibrate the same judgments. Partial receiving selections still owe the selected campaign's full reach matrix. +Another 132 histories use the exported `compileQuery` receiver with the exact +same QueryIR and D2 input streams across two compilations. Both compilation +orders cross shared/separate caches and the 33 source/projection descriptions. +At initial graph drain, 396 states compare every callback capture, complete +derived output and authored selected output. Actual source Collection rows +supply genuine metadata. This receiver establishes the low-level compiler +boundary, not another inline include or optimistic lifecycle witness. + The restored owner reproduced 203 correlated-child REDs on unchanged main. Adding the direct receiving paths before production repair raised that to 348 REDs with 398 passes. The expanded 845-test owner also rejects the first repair in 58 inline QueryRef histories before the corrected production edit. The compiler now requests source/projection metadata at FROM and joined-source boundaries when a functional callback consumes the rows. Inline-only QueryRefs preserve authored shape. -All 847 owner tests pass, including two controls that reject extra named own +All 979 owner tests pass, including two controls that reject extra named own fields on inline arrays. The recorder preserves those fields without changing the model or either judgment. A QueryRef-only mutant leaves 58 joined-source violations; sanitizing every inline child breaks 116 authored-value histories. Restoring unconditional source sanitization breaks 58 inline QueryRef histories. +The extended compiler receiver reproduced 26 shared-comparison failures on +unchanged `9bd623cd5`, with the original 847 tests still passing and all 3,510 +states reached. The cache now keeps results separate by publication mode. +The original raw-query-only cache is the rejected wrong design: authored-first +leaks reserved fields to callbacks, while callback-first loses authored output. +All eight existing compiler-cache controls and the neighboring metadata and +include suites also pass after repair. The [runtime repair record](oracle-reviews/callback-metadata.md) retains the commands, distinguishing traces, source fingerprints and limits. Historical instrument acceptance remains in diff --git a/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md b/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md index 415dab28a4..704a97a3e6 100644 --- a/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md +++ b/docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md @@ -20,8 +20,10 @@ No follow-up PR is claimed until one is actually created. owner has 377 passes / 203 intended REDs and all 2,153 cells; the bounded concrete lane has the same seven failures and one compatible pass on main. The runtime follow-up restores this owner and adds direct child receivers. - It reproduces 348 REDs before repair and passes all 847 tests afterward, with - all 3,114 states reached. Inline QueryRef histories also reject an overbroad + It reproduces 348 REDs before repair and passes all 979 tests afterward, with + all 3,510 states reached. This includes a low-level shared compiler-cache + receiver whose 132 histories reproduced 26 failures before its cache repair. + Inline QueryRef histories also reject an overbroad first repair before the corrected production edit. See [the runtime repair](callback-metadata.md). This does not close the separate static metadata receiving-type failures below. - P11/P12: raw undefined becomes null and selected empty records disappear. diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 1727504330..874167f085 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -30,12 +30,14 @@ Five paths also run directly as the correlated child: 165 histories. Flat union children have no named source reference for include correlation, so their supported forms use the wrapper or union-query-ref path. They remain covered. Three direct-source controls, 132 inline array/materialized histories, 66 compiled -Collection rejections and 19 checker/recorder controls complete 847 tests. Inline +Collection rejections and 19 checker/recorder controls form 847 tests. Inline values cross direct and QueryRef paths. The QueryRef path correlates each parent with one typed-key child, so it preserves authored child fields without a child -functional callback. +functional callback. A later low-level compiler receiver adds 132 histories, +bringing the owner to 979 tests. Its separate initial graph-drain cut is +described below. -Every receiving history observes initial readiness, a held optimistic update, +Each Collection receiving history observes initial readiness, a held optimistic update, settlement without confirmation, and a later remote update. Source rows have numeric `1` and string `"1"` keys and distinct labels. Each callback copies its input immediately, then returns that capture as ordinary output. The comparison @@ -101,8 +103,9 @@ The corrected repair requests source/projection metadata mode when the local query has a functional selection, or when the enclosing query already requires Collection publication mode. It passes that mode into FROM and joined-source compilation. An inline-only QueryRef retains its authored selected shape. -Existing mode parameters and propagation remain. The production diff changes -two call sites. It adds no metadata registry, state, traversal or public API. +Existing mode parameters and propagation remain. That repair changes two +call sites. The later cache repair below separates their compiled results by +mode. Neither adds a metadata registry, traversal or public API. The pre-review head passed all 845 tests. Every one of the 2,520 callback and 594 inline/rejection states reaches once. Seven neighboring suites pass 1,883 tests: root metadata, @@ -243,39 +246,102 @@ The full local snippet scan reports 12 failures in unchanged examples and one stale baseline entry under the reused dependency tree. The earlier CI scan reported only this record's example as a new failure. No dependency install or baseline exclusion was used to mask that local environment difference. Full -final-head snippet verification remains for CI; the changed example passes its -focused local check. +final-head snippet verification then remained for CI. CI on `9bd623cd5` +subsequently passed its full scan: 805 examples, zero new failures, with the +existing baseline unchanged. + +## Shared compiler-cache receiving boundary + +CodeRabbit review `5715750975` on `9bd623cd5` identified a missing cache premise. +The exported `compileQuery` accepts raw QueryIR and an optional cache. Existing +`compiler/subquery-caching.test.ts` controls reuse that cache across calls on +one graph. Cache reuse must preserve the accepted callback-input law and the +authored selected output that separate compilations produce. + +Reusing a QueryBuilder alone did not reach this premise. Ten Effect/Collection +builder probes and two single-root raw-IR probes passed; placement clones or +optimizer lineage gave them distinct effective raw queries. Two compilations +on the same graph/input streams and exact selected QueryIR do reach it: + +| Compilation order with one cache | Wrong public observation | Required observation | +| --- | --- | --- | +| Authored selected root, then downstream callback root | The callback reads `$key: "spoof-$key"` from a structured projection | Reserved fields are absent before the callback | +| Downstream callback root, then authored selected root | The selected output loses authored `$key` | The selected output retains `"spoof-$key"` | + +Separate-cache neighbors pass both orders. The defect was reproduced again +without compiler instrumentation. Earlier scratch shape/literal fixture +mistakes and a diagnostic setup error supply no defect evidence. + +Before production edits, the primary owner added 132 histories: the existing +33 source/projection/collision descriptions, both compilation orders, and +shared/separate caches. Actual public source rows retain numeric `1` and string +`"1"` peers. The existing independent model judges immutable callback captures, +ordinary derived output and raw authored selected output. Signed-bag recording +retains duplicate contributions and retractions. No expected answer comes from +the separate-cache implementation. All three comparisons run before failure. + +Unchanged `9bd623cd5` production gives 953 passes and 26 +`MetadataCallbackOracleFailure` comparisons out of 979 tests. All original +847 tests pass. All 396 new compiler states and all 3,114 prior states reach +once. The driver checks initial graph drain only: its authored-output judgment +is a low-level compiler boundary, not a new real inline include or optimistic +lifecycle witness. Two independent source reviews found no additional defect +in the grammar, recorder, comparisons or proposed cache key. + +The repair retains a result per publication mode under each raw QueryIR key. +Uncorrelated-only admission and same-mode reuse remain. The graph's shared +value-identity scope remains the same. All 979 owner tests pass afterward, +with every one of the 3,510 reach states reached. The original cache is the +hostile wrong design rejected by the 26 REDs. It can be reconstructed by +restoring `QueryCache = WeakMap`, lookup with +`cache.get(rawQuery)`, and storage with `cache.set(rawQuery, compilationResult)`. +No model rule or comparison changes between RED and GREEN. + +Normal package configuration passes all 3,170 targeted tests across ten files, +including this owner, the eight compiler-cache controls, the Effect metadata +owner and the seven neighboring suites named above. Focused no-emit TypeScript, +ESLint, formatting and whitespace checks pass. A temporary RED configuration +only redirects Vite's cache around the reused dependency symlink permissions. +Its jsdom setup, source aliases and shared comparisons are unchanged. The +package-configuration setup failure ran no tests and is excluded. + +This closes the recorded cache-mode violation within the 132 histories and +their initial cut. Arbitrary raw-IR shapes, independent graphs, later input +updates and construction failure need separate evidence. The historical +initial receiving-inventory miss remains documented above. ## Rubric accounting and limits ORC-001–005: accepted authority, independent source facts, literate layers, finite legal grammar, real callback captures and exact derived output cuts. -Removing receiver, callback path, inline path, collision, pending boolean or checkpoint loses a +Removing receiver, callback path, inline path, cache sharing/order, collision, pending boolean or checkpoint loses a named source/projection/inline distinction. Both populated union positions and both typed keys remain represented. Direct-source controls distinguish a blanket stripping rule. The rejected flat-union form bounds admission. ORC-006: 19 hostile observation/recorder controls, the three recorded production -mutants and the independent FROM-only mutant. +mutants, the independent FROM-only mutant and the original wrong cache. ORC-007: exhaustive finite enumeration, so random campaigns and seed replay do -not apply. A valid unrelated `utils.1` seed/path run passes all 845 tests with +not apply. At the pre-review head, a valid unrelated `utils.1` seed/path run passes all 845 tests with all reach intact. A calibration-only selection passes 17 controls without entering a receiver. Selecting only the three direct-source controls fails the missing-matrix reach assertion. An omitted replay property was a configuration error with zero tests, not product evidence. ORC-008–009: source facts distinguish coherent retractions; callback input is distinct from published row, with that abstraction stated beside the model. ORC-010: all-cut -failure reports and failure-preserving cleanup. ORC-011: the concrete shared-cache -hypothesis has six same-path probes; no remaining shared-model hypothesis was -identified. ORC-012: this versioned record and source fingerprints below. +failure reports and failure-preserving cleanup. ORC-011: builder probes missed +the exact raw-query/cache identity premise. The low-level receiver now reaches +that premise in both orders and rejects the wrong cache. ORC-012: this +versioned record and source fingerprints below. ORC-013: single/all/no collisions, typed peers, source/new/inline forms and the three mutants distinguish plausible shortcut repairs. ORC-014: the declared boundary is core runtime compilation, with no real-provider/framework timing -claim. ORC-015: every named callback and inline/rejection state reaches once. +claim. ORC-015: every named callback, inline/rejection and compiler state reaches once. ORC-016: all histories and controls use the shared independent judgments. The runtime evidence is finite: flat inline children, at most two QueryRefs, one LEFT join, two typed-key peers, one held successful mutation, and four cuts. +The separate low-level cache receiver uses one graph and initial graph drain. Arbitrary descriptors/prototypes, deeper recursion, same-key overlapping writes, other join kinds, truncate, framework/provider scheduling and construction-failure cleanup remain outside it. The deferred static metadata receivers and literal @@ -299,7 +365,8 @@ original failures without temporary log files. | Source | SHA256 | | --- | --- | -| `packages/db/src/query/compiler/index.ts` | `07c93e178ddf176af57f85b86479a3f2a762fd96b726f31755e7ed13eafc2fa3` | -| `packages/db/tests/query/virtual-row-fields-input-oracle.ts` | `25f729a6444f7e35526a499aefd7c624acba06348d18f4c31b9d3c2f5d7259a6` | -| `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `c84cf4330e17590b3f9fc55bb449b489a3160e975966823f9ebb4fb4675f18ff` | -| `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `948e84d4f3400ebf8b4e867252580a613dc98e3ebb7eef0811e95727f3624337` | +| `packages/db/src/query/compiler/index.ts` | `a00931f51624d31b4ee749a741fdea30289344a63309b5e2b48b44ee00b913c2` | +| `packages/db/src/query/compiler/types.ts` | `7a8244358fea67f64dac84215adcc079aba4188977b2eb628673632b15c40be6` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.ts` | `d0d526479dda63a01906bcb377675120386e02fe26720ebc7b2c3b7b6e57e658` | +| `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `b24f421e3f5126078d7103ba6ba6a9d7bdd90251c1b4b69ca55e57c020e10292` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `22f075260e1ff8be769409a12b39ce51436c19936d47324d9f866260ab40cfcc` | diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 0989d40c48..e8805380b8 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -437,9 +437,12 @@ function compileQueryInternal( scopeValidated: boolean, publishCollectionRoot = false, ): CompilationResult { - // Check if the original raw query has already been compiled + // Publication modes have different selected-row metadata. Reuse a raw + // query's compilation only within the same mode. const cachedResult = - parentKeyStream === undefined ? cache.get(rawQuery) : undefined + parentKeyStream === undefined + ? cache.get(rawQuery)?.get(publishCollectionRoot) + : undefined if (cachedResult) { return cachedResult } @@ -1329,7 +1332,7 @@ function compileQueryInternal( ) as ResultStream } - // Cache the result before returning (use original query as key) + // Cache the result by the original query and publication mode. const compilationResult: CompilationResult = { collectionId: mainCollectionId, pipeline: resultPipeline, @@ -1339,7 +1342,12 @@ function compileQueryInternal( aliasRemapping, includes: includesResults.length > 0 ? includesResults : undefined, } - if (parentKeyStream === undefined) cache.set(rawQuery, compilationResult) + if (parentKeyStream === undefined) { + getOrCreate(cache, rawQuery, () => new Map()).set( + publishCollectionRoot, + compilationResult, + ) + } return compilationResult } diff --git a/packages/db/src/query/compiler/types.ts b/packages/db/src/query/compiler/types.ts index 83992e512d..3f2dddd6d2 100644 --- a/packages/db/src/query/compiler/types.ts +++ b/packages/db/src/query/compiler/types.ts @@ -4,7 +4,7 @@ import type { CompilationResult } from './index.js' /** * Cache for compiled subqueries to avoid duplicate compilation */ -export type QueryCache = WeakMap +export type QueryCache = WeakMap> /** * Lineage from optimized queries back to their user-defined queries. diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 82b6e0e516..ef59d73db6 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -403,6 +403,10 @@ retain their separate inline shape. The runtime owner checks root, wrapped child and direct child callbacks at initial readiness, held mutation, settlement and remote next use. Its bounded repair evidence is in [the callback metadata record](../../../../../docs/contributing/oracle-reviews/callback-metadata.md). +Reusing a compilation cache on one graph with the same input streams must retain +the callback inputs and authored selected output that separate compilations +produce. The exported `compileQuery` receiver checks both compilation orders +at initial graph drain, separately from the Collection lifecycle receivers. Separate static receiving-type gaps remain in the [deferred oracle record](../../../../../docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md). diff --git a/packages/db/tests/query/virtual-row-fields-input-driver.ts b/packages/db/tests/query/virtual-row-fields-input-driver.ts index 8fa4acd70e..1352ef1c97 100644 --- a/packages/db/tests/query/virtual-row-fields-input-driver.ts +++ b/packages/db/tests/query/virtual-row-fields-input-driver.ts @@ -6,16 +6,20 @@ * may expose an absent joined side. Established output cuts require both rows. * No callback count, ordering, or fully matched startup snapshot is promised. */ -import { inspect } from 'node:util' +import { inspect, isDeepStrictEqual } from 'node:util' +import { D2, MultiSet, output } from '@tanstack/db-ivm' import { createCollection } from '../../src/collection/index.js' import { Query, + compileQuery, createLiveQueryCollection, eq, include, materialize, toArray, } from '../../src/query/index.js' +import { getQueryIR } from '../../src/query/builder/index.js' +import { QueryRef } from '../../src/query/ir.js' import { mockSyncCollectionOptions, withOracleCleanup } from '../utils.js' import { virtualFieldNames } from './virtual-row-fields-support.js' import { @@ -28,6 +32,8 @@ import { import type { Collection } from '../../src/collection/index.js' import type { QueryBuilder } from '../../src/query/builder/index.js' import type { RefLeaf } from '../../src/query/builder/types.js' +import type { QueryCache } from '../../src/query/compiler/types.js' +import type { QueryIR } from '../../src/query/ir.js' import type { MetadataCallbackPath, MetadataInputDescription, @@ -689,3 +695,179 @@ export async function runInlineMetadataCallbackHistory( ]) return report } + +/** The low-level compiler receives the exact same selected QueryIR twice on + * one D2 graph. Unlike builder placement, this public entry point need not + * clone that query. Each history crosses compilation order and cache sharing. + * + * Only the initial graph-drain cut is claimed here. Public source Collection + * rows supply genuine metadata; independent authored facts predict callback + * input and both outputs. The raw selected output uses the same authored-shape + * judgment as an inline value, without claiming an actual include lifecycle. */ +export async function runMetadataCompilerCacheHistory( + description: MetadataInputDescription, + order: `authored-first` | `callback-first`, + cache: `shared` | `separate`, + reached: ( + observation: `callback` | `derived-output` | `authored-output`, + ) => void, +): Promise { + const initial = freshRows() + const first = source(`callback-compiler-cache`, initial) + const facts: Array = initial.map((row) => ({ + row: { ...row }, + sourceId: first.id, + pending: false, + })) + const report: CallbackInputReport = { + violations: [], + callbackCounts: { initial: 0, held: 0, settled: 0, 'remote-next-use': 0 }, + } + await withOracleCleanup(async () => { + await first.preload() + const captures: Array = [] + const selected = getQueryIR(projectInput(first, `chosen`, description)) + const consumed: QueryIR = { + from: new QueryRef(selected, `nested`), + fnSelect: ({ nested }) => { + const observed = captureMetadataInput(nested) + captures.push(observed) + return { id: (nested as MetadataInputRow).id, observed } + }, + } + const graph = new D2() + const input = graph.newInput<[string | number, MetadataInputRow]>() + const sharedCache: QueryCache = new WeakMap() + const compile = (query: QueryIR) => + compileQuery( + query, + { chosen: input }, + { [first.id]: first }, + {}, + {}, + new Set(), + {}, + () => {}, + cache === `shared` ? sharedCache : new WeakMap(), + ) + const [authored, callback] = + order === `authored-first` + ? [compile(selected), compile(consumed)] + : (() => { + const compiledCallback = compile(consumed) + return [compile(selected), compiledCallback] + })() + const authoredDeltas: Array<{ row: Output; weight: number }> = [] + const derivedDeltas: Array<{ row: Output; weight: number }> = [] + authored.pipeline.pipe( + output((message) => { + for (const [[, [value]], weight] of message.getInner()) + authoredDeltas.push({ + row: { + id: (value as MetadataInputRow).id, + observed: captureMetadataInput(value), + }, + weight, + }) + }), + ) + callback.pipeline.pipe( + output((message) => { + for (const [[, [value]], weight] of message.getInner()) { + const row = value as Output + derivedDeltas.push({ + row: { + id: row.id, + // Copy the selected ordinary capture at output delivery too. + observed: copyData(row.observed) as MetadataInputSnapshot, + }, + weight, + }) + } + }), + ) + graph.finalize() + input.sendData(new MultiSet(first.toArray.map((row) => [[row.id, row], 1]))) + graph.run() + + // Independent signed-bag bookkeeping preserves duplicate contributions and + // retractions. It does not replace the source/projection judgment. + const accumulate = ( + deltas: ReadonlyArray<{ row: Output; weight: number }>, + ) => { + const counts: Array<{ row: Output; weight: number }> = [] + for (const delta of deltas) { + const existing = counts.find((entry) => + isDeepStrictEqual(entry.row, delta.row), + ) + if (existing) existing.weight += delta.weight + else counts.push({ ...delta }) + } + const rows: Array = [] + for (const entry of counts) { + if (!Number.isSafeInteger(entry.weight) || entry.weight < 0) { + report.violations.push({ + cut: `initial`, + kind: `invalid compiled output multiplicity`, + actual: entry, + expected: `a nonnegative integer at graph drain`, + }) + continue + } + for (let occurrence = 0; occurrence < entry.weight; occurrence++) + rows.push(entry.row) + } + return rows + } + const actualAuthored = accumulate(authoredDeltas) + const actualDerived = accumulate(derivedDeltas) + const expectedInput = facts.map((fact) => + expectedMetadataInput(fact, description), + ) + const expectedDerived = facts.map((fact, index) => ({ + id: fact.row.id, + observed: expectedInput[index]!, + })) + const expectedAuthored = facts.map((fact) => ({ + id: fact.row.id, + observed: expectedMetadataInput(fact, description, true), + })) + report.callbackCounts.initial = captures.length + if (!captures.length) + report.violations.push({ + cut: `initial`, + kind: `compiler callback not entered`, + actual: captures, + expected: `one or more`, + }) + for (const capture of captures) + for (const kind of metadataInputViolations(capture, expectedInput)) + report.violations.push({ + cut: `initial`, + kind, + actual: capture, + expected: expectedInput, + }) + for (const [name, actual, expected] of [ + [`derived`, actualDerived, expectedDerived], + [`authored`, actualAuthored, expectedAuthored], + ] as const) + for (const kind of metadataOutputViolations(actual, expected)) + report.violations.push({ + cut: `initial`, + kind: `${name}: ${kind}`, + actual, + expected, + }) + // Reach records executed comparisons, including RED comparisons, after the + // real source and both compiled roots reach the named graph-drain cut. + if (first.isReady() && captures.length) reached(`callback`) + if (first.isReady() && actualDerived.length === facts.length) + reached(`derived-output`) + if (first.isReady() && actualAuthored.length === facts.length) + reached(`authored-output`) + if (report.violations.length) + throw new MetadataCallbackOracleFailure(report) + }, [() => first.cleanup()]) + return report +} diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts index 6bf4115875..bf363cb492 100644 --- a/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts @@ -5,6 +5,8 @@ * This finite receiver exhausts 231 root histories, 231 wrapped child histories * and 165 direct child histories. Another 198 inline-value/rejection histories * and three direct-source controls exercise the neighboring boundaries. + * Another 132 histories use the exported compiler with exact shared QueryIR, + * both compilation orders and shared/separate caches at initial graph drain. * It invokes real query callbacks and compares their copied input and ordinary derived output at * initial readiness, held mutation, settlement and later remote use. The * materializer receiver uses the same judgment, not a competing child model. @@ -31,6 +33,7 @@ import { expectedSourceVirtualFields, metadataCallbackHistories, metadataCallbackReceiverHistories, + metadataCompilerCacheHistories, metadataInlineHistories, metadataInputViolations, metadataOutputViolations, @@ -39,6 +42,7 @@ import { captureMetadataInput, runInlineMetadataCallbackHistory, runMetadataCallbackHistory, + runMetadataCompilerCacheHistory, } from './virtual-row-fields-input-driver.js' import type { MetadataSourceFact } from './virtual-row-fields-input-oracle.js' @@ -64,6 +68,28 @@ const inlineCases = metadataInlineHistories.map((history) => ({ `field:`, ), })) +const compilerCases = metadataCompilerCacheHistories.map((history) => ({ + ...history, + label: `${history.order}/${history.cache}/${key(history)}`.replaceAll( + `$`, + `field:`, + ), +})) +const compilerObservations = [ + `callback`, + `derived-output`, + `authored-output`, +] as const +const compilerReach = createOracleReach( + `compiler cache metadata`, + metadataCompilerCacheHistories.flatMap((history) => + compilerObservations.map( + (observation) => + `${history.order}/${history.cache}/${key(history)}/${observation}`, + ), + ), + { replaying: false }, +) const callbackReach = createOracleReach( `functional metadata callback`, [ @@ -91,13 +117,16 @@ const inlineReach = createOracleReach( ) let callbackEntered = false let inlineEntered = false +let compilerEntered = false afterAll(() => { if (callbackEntered) callbackReach.assertReached() if (inlineEntered) inlineReach.assertReached() + if (compilerEntered) compilerReach.assertReached() if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { for (const [name, reach] of [ [`callback`, callbackReach], [`inline`, inlineReach], + [`compiler`, compilerReach], ] as const) { const counts = [...reach.counts().values()] console.info(`${name} metadata reach`, { @@ -115,6 +144,7 @@ declareOracle({ metadataCallbackHistories, metadataCallbackReceiverHistories, metadataInlineHistories, + metadataCompilerCacheHistories, ], }) @@ -177,6 +207,30 @@ test.each(inlineCases)( }, ) +// The initial compiler graph-drain cut is distinct from the Collection +// lifecycle cuts above. Both compilation orders must retain authored output +// while source/projection metadata is judged before downstream JavaScript. +test.each(compilerCases)( + `preserves metadata across compiler cache modes: $label`, + async (history) => { + compilerEntered = true + compilerReach.beginHistory() + const report = await runMetadataCompilerCacheHistory( + history, + history.order, + history.cache, + (observation) => + compilerReach.reach( + `${history.order}/${history.cache}/${key(history)}/${observation}`, + ), + ) + expect( + report.violations, + JSON.stringify(report.violations[0]), + ).toStrictEqual([]) + }, +) + const sourceFact: MetadataSourceFact = { row: { id: 1, parentId: `group`, label: `before`, optional: undefined }, sourceId: `genuine-source`, diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.ts index d764638de6..28de6a76e9 100644 --- a/packages/db/tests/query/virtual-row-fields-input-oracle.ts +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.ts @@ -88,6 +88,20 @@ export const metadataInlineHistories = metadataInputDescriptions.flatMap( })), ), ) +// The exported compiler also accepts an exact QueryIR and an optional cache. +// Reusing one selected query across two roots must preserve both its authored +// output and the source/projection judgment before a downstream callback. +// Separate caches distinguish that promise from an ordinary projection defect. +export const metadataCompilerCacheHistories = metadataInputDescriptions.flatMap( + (description) => + ([`authored-first`, `callback-first`] as const).flatMap((order) => + ([`shared`, `separate`] as const).map((cache) => ({ + ...description, + order, + cache, + })), + ), +) export type MetadataInputSnapshot = { present: boolean ordinary: Record @@ -266,5 +280,6 @@ declareOracle({ metadataCallbackHistories, metadataCallbackReceiverHistories, metadataInlineHistories, + metadataCompilerCacheHistories, ], }) From c63beb064907607ca52093407e27b66a104b6532 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 14:30:26 -0600 Subject: [PATCH 06/19] fix(db): isolate callback metadata and ordered cache windows Law: October 30 maintainer decision extends O12 to select, where and having inputs while retaining authored inline output. Optional caches preserve results. Ordered source loading covers every retained window. Reach: all 22,502 metadata states and 432 added ordered states reached once across the declared finite grammars. RED: extended callback owner rejected 1,102 comparisons before callback edits; ordered owner rejected 48 histories before its registration fix. Real Effect receiver rejects 203 comparisons on PR base. GREEN: 5,912 tests across eight suites; focused TypeScript, ESLint, formatting, whitespace and lint:changed pass. Mutant: WHERE-only 551 failures; all-predicate normalization 696; raw-only cache 26; ordered-plan overwrite 48. All intended comparisons. Limits: finite core source-input/lifecycle witnesses; compiler cache initial drain; union cloning can prevent original-key reuse; controlled ordered compiler/loader cut. Static metadata and P11/P12 gaps remain open. See docs/contributing/oracle-reviews/callback-metadata.md for source bindings and historical process limits. --- .../fix-correlated-callback-metadata.md | 2 +- docs/contributing/oracle-coverage.md | 102 +-- .../oracle-reviews/callback-metadata.md | 233 ++++++- packages/db/src/query/compiler/index.ts | 189 ++++-- packages/db/src/query/compiler/joins.ts | 60 +- packages/db/src/query/compiler/order-by.ts | 36 +- .../db/src/query/compiler/parent-routes.ts | 10 +- .../db/src/query/compiler/route-metadata.ts | 18 + packages/db/src/query/compiler/types.ts | 5 +- packages/db/src/query/live/ARCHITECTURE.md | 35 +- .../query/ordered-compiler-cache-driver.ts | 238 +++++++ .../ordered-work-oracle.property.test.ts | 112 +++- .../query/virtual-row-fields-input-driver.ts | 593 +++++++++++++++--- .../virtual-row-fields-input-oracle.test.ts | 79 ++- .../query/virtual-row-fields-input-oracle.ts | 37 +- 15 files changed, 1482 insertions(+), 267 deletions(-) create mode 100644 packages/db/tests/query/ordered-compiler-cache-driver.ts diff --git a/.changeset/fix-correlated-callback-metadata.md b/.changeset/fix-correlated-callback-metadata.md index be9357ba7c..386e47dbaf 100644 --- a/.changeset/fix-correlated-callback-metadata.md +++ b/.changeset/fix-correlated-callback-metadata.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Fix reserved metadata in functional callback inputs inside correlated includes. Preserve genuine source metadata and authored values when compiled queries share a cache. +Fix reserved metadata before `fn.select()`, `fn.where()`, and `fn.having()` callbacks in live queries, Effects, and correlated includes. Preserve genuine source metadata, authored inline values, and selected output when compiled queries share a cache. Preserve ordered loading for every window when one source feeds multiple compiled views. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 6597550cf0..b43c65d5d7 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -797,54 +797,62 @@ facade-local optimistic mutations. Every shape declares active empty readiness and nonempty child receiving cuts. Six lifetime and four inline-value hostile controls reject wrong observations. See [chunk 5](oracle-reviews/2026-10-28-pr-2171-oracle-chunks.md). Functional callback-input observations have a separate receiving boundary in -`virtual-row-fields-input-oracle.test.ts`, sharing source metadata judgment with -the existing root/Effect owners. Its 33 authoring descriptions cross seven -QueryRef/join/union paths through root and wrapped correlated child compilation -(462 histories), plus five direct correlated child paths (165 histories). -Flat union children require a named QueryRef for include correlation; their -supported forms remain in the wrapped and union-query-ref paths. Three -source controls preserve genuine metadata without an intermediate projection. -Each compares immutable callback captures and ordinary derived output at initial -readiness, held mutation, settlement and remote next use. Complete before/after -source facts permit retraction evaluation. Startup LEFT joins may expose an -absent side; established outputs require both numeric/string-key rows. Reserved -metadata is judged by presence/readable value, with ownership diagnostic. -Ordinary own undefined stays distinct from absence. Another 132 histories retain -nested inline array/materialized authored values through direct and QueryRef -paths, and 66 reject compiled Collection inputs before callbacks. All 2,520 -callback and 594 inline/rejection -states reach once; 19 controls calibrate the same judgments. Partial receiving -selections still owe the selected campaign's full reach matrix. - -Another 132 histories use the exported `compileQuery` receiver with the exact -same QueryIR and D2 input streams across two compilations. Both compilation -orders cross shared/separate caches and the 33 source/projection descriptions. -At initial graph drain, 396 states compare every callback capture, complete -derived output and authored selected output. Actual source Collection rows -supply genuine metadata. This receiver establishes the low-level compiler -boundary, not another inline include or optimistic lifecycle witness. - -The restored owner reproduced 203 correlated-child REDs on unchanged main. -Adding the direct receiving paths before production repair raised that to 348 -REDs with 398 passes. The expanded 845-test owner also rejects the first repair in 58 inline QueryRef -histories before the corrected production edit. The compiler now requests -source/projection metadata at FROM and joined-source boundaries when a functional -callback consumes the rows. Inline-only QueryRefs preserve authored shape. -All 979 owner tests pass, including two controls that reject extra named own -fields on inline arrays. The recorder preserves those fields without changing -the model or either judgment. A QueryRef-only mutant leaves 58 joined-source -violations; sanitizing every inline child breaks 116 authored-value histories. -Restoring unconditional source sanitization breaks 58 inline QueryRef histories. -The extended compiler receiver reproduced 26 shared-comparison failures on -unchanged `9bd623cd5`, with the original 847 tests still passing and all 3,510 -states reached. The cache now keeps results separate by publication mode. -The original raw-query-only cache is the rejected wrong design: authored-first -leaks reserved fields to callbacks, while callback-first loses authored output. -All eight existing compiler-cache controls and the neighboring metadata and -include suites also pass after repair. +`virtual-row-fields-input-oracle.test.ts`, sharing the existing source metadata +judgment. The maintainer extended O12 on 2026-10-30 from fn.select to fn.where +and fn.having while preserving authored inline output. Thirty-three authoring +descriptions cross seven QueryRef/join/union paths, five callback forms +(select, where, having, where-select, having-select), and root, wrapped child, +direct child and real Effect receivers. Flat union children require a named +QueryRef for include correlation; direct flat union children remain inadmissible. +The finite grammar has 4,290 receiving histories and 20 direct-source controls. + +Each receiver checks immutable callback captures and complete derived or +predicate-filtered output at initial readiness, held mutation, settlement and +remote next use. Effects also check exit/reentry, every delivered value and +previousValue, continuity in both callback channels and silence after disposal. +Before/after source facts permit retraction evaluation. Startup LEFT joins may +expose an absent side; after source exit the unmatched peer remains a result. +Reserved fields use presence/readable value, with ownership diagnostic. +Ordinary own undefined remains distinct from absence. Another 132 histories +retain inline array/materialized authored values through direct and QueryRef +paths; 66 reject compiled Collection inputs before callbacks. All 20,720 +callback and 594 inline/rejection states reach once. + +Another 396 histories use exported `compileQuery`, the same exact QueryIR and +D2 input streams, both compilation orders, shared/separate caches, and FROM, +joined-source and union-branch placement. At initial graph drain, all 1,188 +states compare callback captures, complete derived output and raw authored +output. Raw output uses a shared-cache/separate-cache differential, while both +executions still undergo the independent callback/derived-output judgment. +The union optimizer clones branch IR: counters establish original-key reuse +for FROM/JOIN, while union checks propagation/transparency without claiming +that cache hit. This receiver supplies no Effect-root metadata attribution law. + +The owner has 4,923 tests. Before callback production edits, it rejected 1,102 +metadata input comparisons on the reviewed implementation; 3,821 tests passed. +All 22,502 callback, inline and compiler states reached. The repaired owner +passes. Final mutants reject a WHERE-only repair (551 comparisons), enabling +normalization for every predicate pipeline (696 authored-output comparisons), +and dropping the selected-row cache mode (26 comparisons). Earlier path-specific +FROM/JOIN/union normalization mutants each failed 112 compiler comparisons. +The real Effect receiver additionally rejects 203 metadata comparisons on the +actual PR base. That receiving evidence does not retroactively supply the +historical selection repair's initial pre-edit gate. + +The ordered-work owner, `ordered-work-oracle.property.test.ts`, adds 144 finite +histories through `ordered-compiler-cache-driver.ts`. Two compilation orders, +shared/separate caches, FROM/JOIN/union paths, two projection forms, two order +directions and three window moves exercise actual source Collections and the +real OrderedSourceLoader. The independent installed-window model checks signed +output bags at initial, moved and returned cuts. Original registration overwrite +fails 48 histories; the repaired full 424-test owner passes. All 432 new states +reach once. This receiver claims the controlled compiler/loader boundary, not +public Collection window-promise integration. Its 280 existing histories retain +their Collection/Effect receiving evidence. Separate metadata-mode pipelines +remain a permitted representation; no one-pipeline performance promise follows. The [runtime repair record](oracle-reviews/callback-metadata.md) retains the -commands, distinguishing traces, source fingerprints and limits. Historical -instrument acceptance remains in +commands, source bindings, lossless nine-item audit, calibrations and limits. +Historical instrument acceptance remains in [chunk 17](oracle-reviews/2026-10-28-pr-2171-oracle-chunks.md). The separate static receiving-type failures in the [deferred record](oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md) remain open. diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 874167f085..44fec39a80 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -9,7 +9,9 @@ Collection-valued callback inputs reject before user code runs. Authority: the maintainer's O12 decision from 2026-10-28 and the [architecture's callback-input boundary](../../../packages/db/src/query/live/ARCHITECTURE.md#route-context-transport). -This repair enforces that existing runtime law. It revises no product contract. +The original repair enforced the selection-input law. The maintainer extended +it to functional predicates on 2026-10-30, as recorded in the nine-finding +review below. Earlier evidence in this record describes the historical repair. The [coverage owner](../oracle-coverage.md) retains the separate static type gaps. ## Independent owner and receiving histories @@ -370,3 +372,232 @@ original failures without temporary log files. | `packages/db/tests/query/virtual-row-fields-input-oracle.ts` | `d0d526479dda63a01906bcb377675120386e02fe26720ebc7b2c3b7b6e57e658` | | `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `b24f421e3f5126078d7103ba6ba6a9d7bdd90251c1b4b69ca55e57c020e10292` | | `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `22f075260e1ff8be769409a12b39ce51436c19936d47324d9f866260ab40cfcc` | + +## Nine-finding review and predicate extension (2026-10-30) + +Reviewed production: `5ba869f0fc8398243a45a923828b717f0050f40a`. +The implementation and executable evidence below supersede the historical +selection-only counts and source bindings above. The raw nine-item review and +append-only working ledger remain in +`/private/tmp/callback-metadata/nine-issue-review-2312/`. + +### Law, experiment and inference + +The original O12 decision named fn.select. Later architecture prose claimed +functional callbacks generally. The maintainer resolved that conflict on +2026-10-30: extend the input law to select, where and having, while preserving +authored inline output. This is an explicit contract extension. The earlier +selection owner enforced its narrower law but could not generate predicates. + +Direct source rows and structured whole-row source spreads retain genuine +reserved metadata at each source-input callback. A projection with its own +shape exposes none. Callback evaluation must not rewrite the authored result +that an inline child exports. The callback owner derives both views from +independent source facts and projection syntax. Functional predicates actually +derive membership from their captured input, so a spoof can change output. +Predicate-with-selection controls distinguish placement from callback syntax. +Grouped HAVING evaluates its selected group context rather than a local source +namespace; this extension does not revise group attribution or arbitrary +user-created selected-context semantics. + +The extended finite owner has 4,923 tests: 4,290 receiving histories, 20 direct +source controls, 198 inline/rejection histories, 396 compiler histories and 19 +checker controls. The receiving paths are root Collection, wrapped inline +child, direct inline child and real Effect. Each checks initial readiness, +held mutation, settlement and remote next use. Effects also check exit, +reentry, both event channels, every value/previousValue, event continuity and +post-disposal silence. LEFT-join absence has its own model observation. + +The final pre-edit predicate gate produced 1,102 MetadataCallbackOracleFailure +comparisons and 3,821 passes. All 20,720 callback, 594 inline/rejection and 1,188 +compiler states reached once. Initial recorder attempts incorrectly framed +absence and the flat union's `$selected` context. Those setup/evaluation errors +are retained separately and do not supply production RED. Reach now records +callback/host-cut entry independently of output correctness; comparisons remain +strict. Production had only mechanical flag/cache edits and the separately +gated ordered repair before this predicate RED. The callback boundary itself +remained unrepaired. + +The production repair carries a normalized callback view with each row +occurrence beside its authored selected value. FROM, JOIN and union adapters +transport it. Functional callbacks consume that view, then the existing walker +removes private state. Output also removes private state, including for raw +Effect roots. No global object-to-provenance registry merges distinct row +occurrences. Inline output keeps authored values. Selected-row normalization +has an explicit name; its two cache slots replace the per-query boolean Map. +The existing single source-mode expression serves FROM and JOIN. + +The compiler receiver now checks raw selected output by a shared-cache versus +separate-cache relation. It no longer borrows an inline-include attribution +rule for an exported compiler root. Both executions independently check +callback input and derived output, so their agreement cannot excuse a shared +spoof. Its 396 histories cross both orders and all three consuming paths. +Original-key counters distinguish actual FROM/JOIN reuse from optimizer-cloned +union branches. Union coverage establishes propagation/transparency, not an +original-key cache hit. + +A second law requires every ordered window to receive the source rows it +needs. The original source-plan assignment orphaned an earlier top-K: widening +its window from one to three left output `[1]`, while reversed compilation +order produced `[1,2,3]`. The probe recorded two registrations and 15 graph +operators. These are measured work facts, not a universal pipeline budget. +The ordered-work owner now crosses compilation order, cache sharing, FROM/JOIN/ +union placement, projection form, order direction and three window moves: +144 histories and 432 initial/moved/returned states. Its independent existing +installed-window model checks signed output bags. Before the order-by repair, +48 histories failed and 376 tests passed; all old 280 histories passed. +Registration now combines live window bounds and data-needed judgments. It +requires the full source when orders differ. Individual top-K windows and +setters remain independent. All 424 ordered owner tests pass. + +This ordered receiver uses a real source Collection and OrderedSourceLoader +with a controlled subscription/provider seam. It claims the compiler/loader +cut, not public window-promise integration. The existing owner separately +retains its real Collection/Effect lifecycle coverage. + +### Enforcement evidence + +Scratch root: `/private/tmp/callback-metadata/nine-issue-review-2312`. +Commands run with `pnpm_config_verify_deps_before_run=false`, without installs, +builds or lockfile changes, and with at most two Vitest workers. + +| Experiment | Result and evidence | +| --- | --- | +| Unchanged callback production, extended predicate owner | 1,102 failed / 3,821 passed, `predicate-owner-red-corrected.{json,log}` | +| Repaired callback owner | 4,923 passed, `boundary-attempt3.{json,log}` | +| Actual PR base `bfd1da1a7e752d66fd667adfd3cdf54f60732b22` with new Effect receiver | 203 Effect comparisons failed, `pr-base-owner.{json,log}` | +| Original ordered source registration | 48 failed / 376 passed, `ordered-owner-red.{json,log}` | +| Repaired ordered owner | 424 passed, `ordered-owner-green.{json,log}` | +| Normal package configuration, eight neighboring suites | 5,912 passed, `final-package-green.{json,log}` | +| WHERE-only callback repair | 551 HAVING comparisons failed, `final-where-only.{json,log}` | +| Normalize every predicate's source pipeline | 696 authored-output comparisons failed, `final-sanitize-all-predicates.{json,log}` | +| Lose selected-row cache mode | 26 compiler comparisons failed, `final-raw-only-cache.{json,log}` | +| Restore source-plan overwrite | 48 ordered comparisons failed, `final-overwrite-window.{json,log}` | +| Earlier FROM/JOIN/union mode omissions | 112 compiler comparisons failed per path, `mutant-summary.json` | + +The final mutants ran against copied final sources and final tests. They failed +at the intended comparisons, rather than during setup. Their source edits and +full failure traces remain in `run-final-mutants.py`, `final-mutant-baseline/` +and `final-mutant-summary.json`. The original predicate RED also rejects the +reviewed implementation as a hostile wrong design. No comparison, reach +minimum or run budget was lowered. + +### Lossless item audit + +| ID | Technical verdict | Action and durable value | +| --- | --- | --- | +| EXT2312-01 | Confirmed predicate input spoof for WHERE and HAVING, including real Effects and inline children. The original select-only authority required the maintainer's resolved extension. | fixed-now: occurrence callback view plus callback-kind grammar, membership and authored-output checks. | +| EXT2312-02 | Confirmed broad prose exceeded the implemented and generated select-only promise. No predicate comparison existed to enforce it. | fixed-now: approved law, executable grammar and current architecture/coverage claims agree. Historical initial-gate gaps remain explicit. | +| EXT2312-03 | Confirmed real Effect receiver was missing. Its actual PR-base run rejects 203 metadata comparisons. | fixed-now: shared-model Effect lifecycle receiver and corrected changeset. New evidence does not retroactively repair the original pre-edit process. | +| EXT2312-04 | Confirmed absolute raw-root judgment lacked an independent attribution authority. | fixed-now: differential raw-output law, retaining independent input/output checks in both executions. | +| EXT2312-05 | Confirmed two mode pipelines and the orphaned ordered plan, with an actual underfilled result. Pipeline duplication alone violates no stated resource law. | fixed-now: primary ordered owner plus live aggregate source plan. Separate top-Ks remain permitted. No unsupported one-pipeline or speed claim. | +| EXT2312-06 | Confirmed duplicate expression at FROM/JOIN. | fixed-now: compute the expression once. Path-specific mutation evidence preserves the reason. | +| EXT2312-07 | Confirmed ambiguous publication name. The cited branches normalize selected metadata rather than performing Collection publication. | fixed-now: rename to sanitizeSelectedMetadata and separate callback view transport. No independent root-only behavior defect follows from the old name. | +| EXT2312-08 | Confirmed a Map represented only two slots. No heap or bundle measurement supported the claimed magnitude. | fixed-now: typed two-slot tuple. Claim only the removed Map structure, not measured bytes or speed. | +| EXT2312-09 | Confirmed FROM-only cache grammar omitted JOIN and union. | fixed-now: three-path grammar and path-specific mutant sensitivity. Preserve the union optimizer-cloning limit. | + +All nine raw items have separate IDs, claims, proposed repairs, evidence and +dispositions in the append-only ledger. Total: nine fixed-now, zero deferred, +zero unresolved product decisions. The ordered-loader counterexample is the +confirmed consequence of item 5, rather than an extra invented raw item. +The initial repair attempt exposed private callback-view state in raw Effect +output; final result cleanup removes it, and Effect continuity rejects that +attempt. This collateral attempt is retained as implementation feedback. + +The reviewer earns a strong hire recommendation. The predicate probes identified +a real placement-dependent leak, and source analysis predicted the ordered-plan +failure that execution confirmed. The differential and path-dimension proposals +improve test integrity. The flag and allocation comments are useful cleanup, +with lower severity than the correctness findings. The review did not establish +a performance budget or an original-key union cache hit. Those qualifications +limit the claims without discarding the findings. + +### Guide audit and limits + +ORC-001–003: the approved law, limits, independent model, grammar, real receivers +and comparisons remain beside their executable definitions. ORC-004: existing +33 legal descriptions retain all single/all/no collisions and typed peers; +callback-kind ablation loses predicates, selection controls lose placement +independence, and path ablation loses a distinct transport boundary. Compiled +Collection inputs and direct-return overrides remain excluded by admission. +ORC-005: each receiving cut and every delivered Effect value is compared. +ORC-006: the recorded original and final mutants fail their intended judgments. +ORC-007: callback/cache additions exhaust finite grammars, so seed campaigns do +not apply; existing ordered generated campaigns keep fixed/random parity. +ORC-008–009: source facts distinguish retractions, callback snapshots differ +from Collection rows, and the controlled ordered seam has an explicit mapping. +ORC-010: all-cut reports and cleanup preserve primary failures. ORC-011: +cache transparency uses two executions while independent input truth prevents +a shared false green. ORC-012: this versioned record binds the final source +hashes below. ORC-013: WHERE-only and over-sanitizing repairs demonstrate the +neighboring consequences. ORC-014: actual Collection/Effect receivers supply +core lifecycle evidence; ordered provider and framework claims stay bounded. +ORC-015: all 22,502 metadata and 432 ordered added states reach once. +ORC-016: all added histories use shared judgments, without pinned expected +answers or unit tests inside oracle files. + +The finite metadata grammar bounds recursion at two QueryRefs, joins at one +LEFT join, sources at two typed-key peers, and mutations at one held successful +update. It does not establish arbitrary descriptors/prototypes, deeper +recursion, overlapping same-key mutations, other join kinds, truncate, +provider scheduling, framework timing or construction-failure cleanup. +Grouped selected-context attribution is a separate judgment from the local +source-input predicate law. The existing aggregate and materialization owners +remain responsible for those semantics. Static metadata and P11/P12 literal/ +type gaps stay open in the retained deferred record. No universal runtime or +static closure follows from the finite evidence here. + +### Final source binding and verification + +The final package run passes all 5,912 tests across eight suites. Focused +TypeScript passes with no diagnostics. Explicit ESLint passes with zero errors +and four existing no-shadow warnings in the ordered owner. Formatting, +whitespace and changed-file oracle enforcement pass. The final four mutants +were refreshed from these exact compiler and test sources and again rejected +551, 696, 26 and 48 intended comparisons respectively. Their logs retain all +failure messages. These results supersede intermediate repair attempts. + +Run both owners under the normal package configuration: + +```sh +TANSTACK_DB_ORACLE_STATISTICS=1 pnpm_config_verify_deps_before_run=false \ + pnpm --dir packages/db exec vitest run --maxWorkers=2 \ + --coverage.enabled=false --typecheck.enabled=false \ + tests/query/virtual-row-fields-input-oracle.test.ts \ + tests/query/ordered-work-oracle.property.test.ts +``` + +The predicate RED command used the extended tests and the pre-edit compiler +binding in `predicate-gate-fingerprints.json`: + +```sh +pnpm_config_verify_deps_before_run=false pnpm exec vitest run \ + --config /private/tmp/callback-metadata/nine-issue-review-2312/owner.config.mjs \ + --reporter=json \ + --outputFile=/private/tmp/callback-metadata/nine-issue-review-2312/predicate-owner-red-corrected.json +``` + +The ordered RED used the extended ordered owner and the pre-edit binding in +`ordered-gate-fingerprints.json`, with `ordered-original.config.mjs`. The final +hostile calibrations are reproducible through `run-final-mutants.py`. The +normal eight-suite command adds the existing Effect metadata, cache-control, +functional projection/input boundary, public-container-copy and context +transport suites to the two-owner command above. No full build or dependency +installation supplied this evidence. + +These SHA-256 fingerprints bind the final behavioral sources without making +the report hash itself refer to its own contents: + +| Source | SHA-256 | +| --- | --- | +| `packages/db/src/query/compiler/index.ts` | `c566eccd683862fb3235a84e841fafca32fb8c5a43491570b2fa65a3c1874487` | +| `packages/db/src/query/compiler/joins.ts` | `1c3642f632a10b25ac784d5728e680803d657452969496a4e21b1a8213f44fcd` | +| `packages/db/src/query/compiler/order-by.ts` | `384d8a132c86265f4bb4e12ec4ed5200f062a182e35402dcd38b4d546828d0d2` | +| `packages/db/src/query/compiler/parent-routes.ts` | `5f8caf4504d70639bcd9b109da28ddb6880ebc81b1e4489e0ae8de2e7151cf42` | +| `packages/db/src/query/compiler/route-metadata.ts` | `ba0c42104288720ff221b516298d39da9e7201d68984f8101adb7df20f286cb1` | +| `packages/db/src/query/compiler/types.ts` | `e06dbdf363cc1878226692de890650680e1c84bf35f17424fd23dad09313c27d` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.ts` | `7b7c5055fe58dd701ad10db9620f516d1ab4b0127a95e17bee629310b3b5450a` | +| `packages/db/tests/query/virtual-row-fields-input-driver.ts` | `3d157039ec6c7aead891865738346e4e52c3747fe199fca44fce0224ed4998ad` | +| `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `58ae2e54947145220a8b78a28cadc37456e3d11d507ce36846843f002d79550d` | +| `packages/db/tests/query/ordered-compiler-cache-driver.ts` | `ba7b244250754d9a5d8480847562c26907edf192c8a028973dca0de32e6c43e6` | +| `packages/db/tests/query/ordered-work-oracle.property.test.ts` | `a8e2f5884570f0659764ad0b7be7398c432b411fcf796dd007ddd3f4bb9d5e68` | diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index e8805380b8..a7c1964593 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -75,16 +75,20 @@ import { processOrderBy } from './order-by.js' import { crossJoinParentRoutes } from './parent-routes.js' import { queriesMatchForCaching } from './query-equivalence.js' import { + CALLBACK_INPUT, + CALLBACK_SELECTED, INCLUDES_PUBLIC_KEY, INCLUDES_ROUTING, attachRouteMetadata, attachRouteMetadataToResult, + getCallbackInput, getNamespacedRouteMetadata, getRouteMetadata, getRoutedScalarMetadata, stripInternalCallbackMetadata, stripInternalRouteMetadata, stripRouteMetadata, + transformPublicContainers, } from './route-metadata.js' import { processSelect } from './select.js' import type { ValueIdentity } from '../equality-value-identity.js' @@ -372,6 +376,8 @@ export interface CompilationResult { includes?: Array } +const NO_OMITTED_KEYS: ReadonlySet = new Set() + const valueIdentitiesByCache = new WeakMap() function getCompilationValueIdentity(cache: QueryCache): ValueIdentity { @@ -435,13 +441,13 @@ function compileQueryInternal( ancestorHasTopK = false, ]: CompileQueryArgs, scopeValidated: boolean, - publishCollectionRoot = false, + sanitizeSelectedMetadata = false, ): CompilationResult { - // Publication modes have different selected-row metadata. Reuse a raw + // Authored and source views have different selected-row metadata. Reuse a raw // query's compilation only within the same mode. const cachedResult = parentKeyStream === undefined - ? cache.get(rawQuery)?.get(publishCollectionRoot) + ? cache.get(rawQuery)?.[sanitizeSelectedMetadata ? 1 : 0] : undefined if (cachedResult) { return cachedResult @@ -463,6 +469,8 @@ function compileQueryInternal( ) // Use a mutable binding so we can shallow-clone select before includes mutation let query = optimizedQuery + const sanitizeSourceMetadata = + sanitizeSelectedMetadata || query.fnSelect !== undefined // Create mapping from optimized query to original for caching queryMapping.set(query, rawQuery) @@ -514,7 +522,7 @@ function compileQueryInternal( parentKeyStream, // Sanitize source rows before a callback consumes them. Inline values // without a callback keep their authored selected shape. - publishCollectionRoot || query.fnSelect !== undefined, + sanitizeSourceMetadata, ) Object.assign(sources, fromSources) const sourceCarriesInternalRouteState = @@ -541,12 +549,12 @@ function compileQueryInternal( // child value beside the row so result routing can still expose it. const childFieldPath = childCorrelationField.path.slice(1) // remove alias prefix const childRekeyed = mainInput.pipe( - map(([key, row]: [unknown, any]) => { + map(([key, row, callbackValue]: [...InputRow, unknown?]) => { const correlationValue = getNestedValue(row, childFieldPath) return [ valueIdentity.serializeEquality(correlationValue), - [key, row, correlationValue], - ] as [unknown, [unknown, any, unknown]] + [key, row, correlationValue, callbackValue], + ] as [unknown, [unknown, any, unknown, unknown?]] }), ) @@ -573,7 +581,7 @@ function compileQueryInternal( return childSide != null }), map(([_correlationIdentity, [childSide, parentSide]]: any) => { - const [childKey, childRow, correlationValue] = childSide + const [childKey, childRow, correlationValue, callbackValue] = childSide const tagged: any = attachRouteMetadata( { ...childRow, @@ -589,7 +597,7 @@ function compileQueryInternal( getParentContextIdentity(parentSide), ]) : childKey - return [effectiveKey, tagged] + return [effectiveKey, tagged, callbackValue] as unknown as InputRow }), ) @@ -627,12 +635,7 @@ function compileQueryInternal( ignoreWindow, rawQuery, query.from, - (...args) => - compileQueryInternal( - args, - true, - publishCollectionRoot || query.fnSelect !== undefined, - ), + (...args) => compileQueryInternal(args, true, sanitizeSourceMetadata), aliasToCollectionId, aliasRemapping, sourceWhereClauses, @@ -683,9 +686,9 @@ function compileQueryInternal( for (const fnWhere of query.fnWhere) { pipeline = pipeline.pipe( filter(([_key, namespacedRow]) => { - const callbackRow = sourceCarriesInternalRouteState - ? (stripInternalCallbackMetadata(namespacedRow) as NamespacedRow) - : namespacedRow + const callbackRow = stripInternalCallbackMetadata( + getCallbackInput(namespacedRow), + ) as NamespacedRow return toBooleanPredicate(fnWhere(callbackRow)) }), ) @@ -1061,9 +1064,9 @@ function compileQueryInternal( const sourceAliases = [...sourceBindings.keys()] // Handle functional select - apply the function to transform the row const projectRow = (namespacedRow: NamespacedRow) => { - const callbackRow = sourceCarriesInternalRouteState - ? (stripInternalCallbackMetadata(namespacedRow) as NamespacedRow) - : namespacedRow + const callbackRow = stripInternalCallbackMetadata( + getCallbackInput(namespacedRow), + ) as NamespacedRow const selectResults = fnSelect(callbackRow) let selected = selectResults if ( @@ -1075,7 +1078,7 @@ function compileQueryInternal( ? [...selectResults] : { ...selectResults } if ( - publishCollectionRoot && + sanitizeSelectedMetadata && !sourceAliases.some((alias) => selectResults === callbackRow[alias]) ) { // New functional objects are projections. Equal field values cannot @@ -1087,9 +1090,24 @@ function compileQueryInternal( selected[INCLUDES_ROUTING] = routing } } + let callbackSelected = selected + if ( + selected && + typeof selected === `object` && + (Array.isArray(selected) || isPlainObject(selected)) && + !sourceAliases.some((alias) => selectResults === callbackRow[alias]) + ) { + for (const prop of VIRTUAL_PROP_NAMES) { + if (!(prop in selected)) continue + if (callbackSelected === selected) + callbackSelected = copySelectedRow(selected) + delete callbackSelected[prop] + } + } return { ...namespacedRow, $selected: selected, + [CALLBACK_SELECTED]: callbackSelected, } } pipeline = pipeline.pipe(map(([key, row]) => [key, projectRow(row)])) @@ -1199,10 +1217,9 @@ function compileQueryInternal( for (const fnHaving of query.fnHaving) { pipeline = pipeline.pipe( filter(([_key, namespacedRow]) => { - const callbackRow = - sourceCarriesInternalRouteState || includesRoutingFns.length > 0 - ? (stripInternalCallbackMetadata(namespacedRow) as NamespacedRow) - : namespacedRow + const callbackRow = stripInternalCallbackMetadata( + getCallbackInput(namespacedRow), + ) as NamespacedRow return fnHaving(callbackRow) }), ) @@ -1240,32 +1257,59 @@ function compileQueryInternal( row: Record, orderByIndex: string | undefined, ) => { - const finalResults = - query.fnSelect && publishCollectionRoot + const authoredResults = + query.fnSelect && sanitizeSelectedMetadata ? unwrapValue(row.$selected) : attachVirtualPropsToSelected( unwrapValue(row.$selected), row, - publishCollectionRoot, + sanitizeSelectedMetadata, metadataSource, ) - // When in includes mode, embed the correlation key and parentContext + const finalResults = stripInternalRouteMetadata(authoredResults) + const callbackRow = getCallbackInput(row) + const sourceViews = Object.entries(callbackRow).filter( + ([alias, value]) => alias !== `$selected` && value !== row[alias], + ) + const callbackSelected = query.fnSelect + ? (row as { [CALLBACK_SELECTED]?: unknown })[CALLBACK_SELECTED] + : attachVirtualPropsToSelected( + transformPublicContainers( + unwrapValue(row.$selected), + (value) => + sourceViews.find(([alias]) => value === row[alias])?.[1] ?? value, + NO_OMITTED_KEYS, + ), + callbackRow, + true, + metadataSource, + ) + // Carry both views with the occurrence. A downstream source adapter selects + // the callback view without rewriting the authored inline result. if (parentKeyStream) { return [ key, [ - stripInternalRouteMetadata(finalResults), + finalResults, orderByIndex, getRowCorrelationKey(row, mainSource), getRowParentContext(row, mainSource), getIncludesPublicKey(row, mainSource, key), + stripInternalRouteMetadata(callbackSelected), ], ] as any } - return [key, [finalResults, orderByIndex]] as [ - unknown, - [any, string | undefined], - ] + return [ + key, + [ + finalResults, + orderByIndex, + undefined, + undefined, + undefined, + stripInternalRouteMetadata(callbackSelected), + ], + ] as unknown as [unknown, [any, string | undefined]] } // A single-result query must expose only its first row when another query @@ -1332,7 +1376,7 @@ function compileQueryInternal( ) as ResultStream } - // Cache the result by the original query and publication mode. + // Cache the result by the original query and selected-row metadata mode. const compilationResult: CompilationResult = { collectionId: mainCollectionId, pipeline: resultPipeline, @@ -1343,10 +1387,11 @@ function compileQueryInternal( includes: includesResults.length > 0 ? includesResults : undefined, } if (parentKeyStream === undefined) { - getOrCreate(cache, rawQuery, () => new Map()).set( - publishCollectionRoot, - compilationResult, - ) + getOrCreate( + cache, + rawQuery, + (): [CompilationResult?, CompilationResult?] => [], + )[sanitizeSelectedMetadata ? 1 : 0] = compilationResult } return compilationResult @@ -1612,7 +1657,7 @@ function processFromClause( aliasRemapping: Record, sourceWhereClauses: Map>, parentKeyStream?: KeyedStream, - publishCollectionRoot = false, + sanitizeSelectedMetadata = false, ): { alias: string pipeline: NamespacedAndKeyedStream @@ -1640,7 +1685,7 @@ function processFromClause( aliasRemapping, sourceWhereClauses, parentKeyStream, - publishCollectionRoot, + sanitizeSelectedMetadata, ) } @@ -1661,7 +1706,7 @@ function processFromClause( aliasRemapping, sourceWhereClauses, parentKeyStream, - publishCollectionRoot, + sanitizeSelectedMetadata, ) return { @@ -1708,7 +1753,7 @@ function processFromClause( aliasRemapping, sourceWhereClauses, parentKeyStream, - publishCollectionRoot, + sanitizeSelectedMetadata, ) if (!mainAlias) { @@ -1777,7 +1822,7 @@ function processUnionAll( aliasRemapping: Record, sourceWhereClauses: Map>, parentKeyStream?: KeyedStream, - publishCollectionRoot = false, + sanitizeSelectedMetadata = false, ): { alias: string pipeline: NamespacedAndKeyedStream @@ -1828,7 +1873,7 @@ function processUnionAll( undefined, ], true, - publishCollectionRoot, + sanitizeSelectedMetadata, ) if (!mainCollectionId) { @@ -1844,8 +1889,17 @@ function processUnionAll( const branchPipeline = branchResult.pipeline.pipe( map((data: any) => { - const [key, [row, _order, correlationKey, parentContext, publicKey]] = - data + const [ + key, + [ + row, + _order, + correlationKey, + parentContext, + publicKey, + callbackValue, + ], + ] = data const branchKey = `${index}:${encodeKeyForUnionBranch(key)}` const branchPublicKey = `${index}:${encodeKeyForUnionBranch( publicKey ?? key, @@ -1858,7 +1912,13 @@ function processUnionAll( branchPublicKey, ) : row - return [branchKey, routedRow] as [string, Record] + const namespaced = + callbackValue !== undefined && + callbackValue !== row && + isPlainObject(routedRow) + ? { ...routedRow, [CALLBACK_INPUT]: [callbackValue] } + : routedRow + return [branchKey, namespaced] as [string, Record] }), ) @@ -1884,7 +1944,15 @@ function wrapInputWithAlias( alias: string, ): NamespacedAndKeyedStream { return input.pipe( - map(([key, row]) => { + map(([key, row, callbackValue]: [...InputRow, unknown?]) => { + const finish = (namespaced: NamespacedRow): [unknown, NamespacedRow] => { + if (callbackValue !== undefined && callbackValue !== row) { + Object.assign(namespaced, { + [CALLBACK_INPUT]: { [alias]: callbackValue }, + }) + } + return [key, namespaced] + } const inputRow: unknown = row const scalar = getRoutedScalarMetadata(inputRow) if (scalar) { @@ -1897,11 +1965,11 @@ function wrapInputWithAlias( scalar.correlationKey, scalar.parentContext, ) as unknown as NamespacedRow - return [key, nsRow] as [unknown, NamespacedRow] + return finish(nsRow) } if (inputRow == null || typeof inputRow !== `object`) { - return [key, { [alias]: inputRow }] as [unknown, NamespacedRow] + return finish({ [alias]: inputRow } as NamespacedRow) } // Initialize the record with a nested structure. Route metadata remains @@ -1917,7 +1985,7 @@ function wrapInputWithAlias( if (route) { attachRouteMetadata(nsRow, route.correlationKey, route.parentContext) } - return [key, nsRow] as [unknown, Record] + return finish(nsRow) }), ) } @@ -1950,7 +2018,7 @@ function processFrom( aliasRemapping: Record, sourceWhereClauses: Map>, parentKeyStream?: KeyedStream, - publishCollectionRoot = false, + sanitizeSelectedMetadata = false, ): { alias: string input: KeyedStream @@ -2004,7 +2072,7 @@ function processFrom( undefined, ], true, - publishCollectionRoot, + sanitizeSelectedMetadata, ) // Pull up alias mappings from subquery to parent scope. @@ -2065,7 +2133,14 @@ function processFrom( map((data: any) => { const [ key, - [value, _orderByIndex, correlationKey, parentContext, publicKey], + [ + value, + _orderByIndex, + correlationKey, + parentContext, + publicKey, + callbackValue, + ], ] = data // Unwrap Value expressions that might have leaked through as the entire row const unwrapped = attachRouteMetadataToResult( @@ -2074,7 +2149,7 @@ function processFrom( parentContext, publicKey, ) - return [key, unwrapped] as [unknown, any] + return [key, unwrapped, callbackValue] as unknown as InputRow }), ) diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 65f1e38092..619e4f7b2e 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -32,9 +32,11 @@ import { getLazyLoadTargets } from './lazy-targets.js' import { crossJoinParentRoutes } from './parent-routes.js' import { queriesMatchForCaching } from './query-equivalence.js' import { + CALLBACK_INPUT, INCLUDES_PUBLIC_KEY, attachRouteMetadata, attachRouteMetadataToResult, + getCallbackInput, getNamespacedRouteMetadata, getRouteMetadata, getRoutedScalarMetadata, @@ -55,6 +57,7 @@ import type { import type { IStreamBuilder, JoinType } from '@tanstack/db-ivm' import type { Collection } from '../../collection/index.js' import type { + InputRow, KeyedStream, NamespacedAndKeyedStream, NamespacedRow, @@ -91,7 +94,7 @@ function parameterizeJoinInputByParentRoutes( return crossJoinParentRoutes( input, parentKeyStream, - (rowKey, row, correlationKey, parentContext) => { + (rowKey, row, correlationKey, parentContext, callbackValue) => { return [ serializeValue([ valueIdentity.equality(rowKey), @@ -105,12 +108,24 @@ function parameterizeJoinInputByParentRoutes( correlationKey, parentContext, ), - ] + callbackValue, + ] as unknown as [unknown, unknown] }, ) } -function wrapJoinedInputRow(alias: string, row: any): NamespacedRow { +function wrapJoinedInputRow( + alias: string, + row: any, + callbackValue?: unknown, +): NamespacedRow { + const finish = (namespaced: NamespacedRow): NamespacedRow => { + if (callbackValue !== undefined && callbackValue !== row) + Object.assign(namespaced, { + [CALLBACK_INPUT]: { [alias]: callbackValue }, + }) + return namespaced + } const scalar = getRoutedScalarMetadata(row) if (scalar) { const namespaced = attachRouteMetadata( @@ -122,11 +137,11 @@ function wrapJoinedInputRow(alias: string, row: any): NamespacedRow { scalar.correlationKey, scalar.parentContext, ) as unknown as NamespacedRow - return namespaced + return finish(namespaced) } if (row == null || typeof row !== `object`) { - return { [alias]: row } + return finish({ [alias]: row } as NamespacedRow) } const route = getRouteMetadata(row) @@ -138,7 +153,7 @@ function wrapJoinedInputRow(alias: string, row: any): NamespacedRow { if (route) { attachRouteMetadata(namespaced, route.correlationKey, route.parentContext) } - return namespaced + return finish(namespaced) } function getRouteJoinKey( @@ -421,10 +436,10 @@ function processJoin( ) let joinedPipeline = joinedInput.pipe( - map(([currentKey, row]) => + map(([currentKey, row, callbackValue]: [...InputRow, unknown?]) => keyJoinInput( currentKey, - wrapJoinedInputRow(joinedSource, row), + wrapJoinedInputRow(joinedSource, row, callbackValue), joinedSource, `joined`, compiledJoinedExpressions, @@ -738,10 +753,17 @@ function processJoinSource( map((data: any) => { const [ key, - [value, _orderByIndex, correlationKey, parentContext, publicKey], + [ + value, + _orderByIndex, + correlationKey, + parentContext, + publicKey, + callbackValue, + ], ] = data if (!parentKeyStream) { - return [key, value] as [unknown, any] + return [key, value, callbackValue] as unknown as InputRow } return [ key, @@ -751,7 +773,8 @@ function processJoinSource( parentContext, publicKey, ), - ] as [unknown, any] + callbackValue, + ] as unknown as InputRow }), ) @@ -813,6 +836,21 @@ function processJoinResults( delete mergedNamespacedRow[joinedSource] } + const mainView = + mainNamespacedRow && getCallbackInput(mainNamespacedRow) + const joinedView = + joinedNamespacedRow && getCallbackInput(joinedNamespacedRow) + if ( + mainView !== mainNamespacedRow || + joinedView !== joinedNamespacedRow + ) { + const view = { ...(mainView ?? joinedView) } as NamespacedRow + if (main == null) for (const alias of mainAliases) delete view[alias] + if (joinedView) view[joinedSource] = joinedView[joinedSource]! + else delete view[joinedSource] + Object.assign(mergedNamespacedRow, { [CALLBACK_INPUT]: view }) + } + // Combine the main and joined keys without ambiguity: keys may contain // delimiters, numbers and strings may print alike, and a missing outer // side encodes as null, which no source key can be. diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index 518a0fddad..f0c05baae3 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -17,6 +17,7 @@ import { supportsOrderedRange, } from '../../utils/index-optimization.js' import { hasVirtualPropPath } from '../../virtual-props.js' +import { deepEquals } from '../../utils.js' import { devBuild } from '../../error-message.js' import { compileExpression } from './evaluators.js' import { getSourceAliasesFromExpression } from './expressions.js' @@ -332,7 +333,40 @@ export function processOrderBy( // Ordered loading is owned by one lexical source. A collection can occur // more than once in a query tree, so collection ID and alias are not // sufficient identities here. - optimizableOrderByCollections[orderBySourceId] = info + const previous = optimizableOrderByCollections[orderBySourceId] + // One lexical source can feed separate authored/source metadata views. + // Each top-K owns its own window. Acquisition must cover all windows, + // including moves made through a previously compiled root's callback. + // Keep these reads live: setWindowFn updates the individual info objects. + const sameOrder = previous && deepEquals(previous.orderBy, info.orderBy) + optimizableOrderByCollections[orderBySourceId] = previous + ? { + ...info, + get offset() { + return Math.min(previous.offset, info.offset) + }, + get limit() { + return ( + Math.max( + previous.offset + previous.limit, + info.offset + info.limit, + ) - Math.min(previous.offset, info.offset) + ) + }, + get requiresFullSource() { + return ( + !sameOrder || + previous.requiresFullSource || + info.requiresFullSource + ) + }, + dataNeeded: (boundary) => + Math.max( + previous.dataNeeded?.(boundary) ?? 0, + info.dataNeeded?.(boundary) ?? 0, + ), + } + : info // loadMoreIfNeeded reads how many rows the window needs. Without an // index, only an on-demand joined filter can leave the window short. diff --git a/packages/db/src/query/compiler/parent-routes.ts b/packages/db/src/query/compiler/parent-routes.ts index b6eddb32f9..1a28b7c2af 100644 --- a/packages/db/src/query/compiler/parent-routes.ts +++ b/packages/db/src/query/compiler/parent-routes.ts @@ -11,13 +11,17 @@ export function crossJoinParentRoutes( row: unknown, correlationKey: unknown, parentContext: unknown, + callbackValue?: unknown, ) => [unknown, unknown], ): KeyedStream { // Recursive sources need their route before a correlation field is always // available. The constant key intentionally creates one copy of each input // row per active route; callers filter those copies once the field is visible. const rows: any = input.pipe( - map(([rowKey, row]) => [PARENT_ROUTE_CROSS_KEY, [rowKey, row]]), + map(([rowKey, row, callbackValue]: [unknown, unknown, unknown?]) => [ + PARENT_ROUTE_CROSS_KEY, + [rowKey, row, callbackValue], + ]), ) const routes: any = parentKeyStream.pipe( map(([correlationKey, parentContext]) => [ @@ -32,9 +36,9 @@ export function crossJoinParentRoutes( Boolean(rowSide != null && routeSide != null), ), map(([, [rowSide, routeSide]]: any) => { - const [rowKey, row] = rowSide + const [rowKey, row, callbackValue] = rowSide const [correlationKey, parentContext] = routeSide - return assemble(rowKey, row, correlationKey, parentContext) + return assemble(rowKey, row, correlationKey, parentContext, callbackValue) }), ) as KeyedStream } diff --git a/packages/db/src/query/compiler/route-metadata.ts b/packages/db/src/query/compiler/route-metadata.ts index 9a5fad30d0..c3627659c5 100644 --- a/packages/db/src/query/compiler/route-metadata.ts +++ b/packages/db/src/query/compiler/route-metadata.ts @@ -1,12 +1,16 @@ import { isPlainObject } from '../../utils/type-guards.js' const ROUTED_SCALAR_VALUE = Symbol(`tanstack_db_routed_scalar_value`) +export const CALLBACK_INPUT = Symbol(`tanstack_db_callback_input`) +export const CALLBACK_SELECTED = Symbol(`tanstack_db_callback_selected`) const ROUTE_METADATA = Symbol(`tanstack_db_route_metadata`) export const INCLUDES_PUBLIC_KEY = Symbol(`includesPublicKey`) export const INCLUDES_ROUTING = Symbol(`includesRouting`) const INTERNAL_ROUTE_KEYS = new Set([ ROUTE_METADATA, INCLUDES_PUBLIC_KEY, + CALLBACK_INPUT, + CALLBACK_SELECTED, ]) const INTERNAL_CALLBACK_KEYS = new Set([ ...INTERNAL_ROUTE_KEYS, @@ -125,6 +129,20 @@ export function getRoutedScalarMetadata( } } +/** A namespace keeps authored values beside its callback source overrides. + * A one-element tuple denotes a flat union view: absent fields must disappear, + * whereas named overrides replace only their own source aliases. + */ +export function getCallbackInput(row: T): T { + const view = (row as Record)[CALLBACK_INPUT] + if (Array.isArray(view)) { + const result = { ...view[0] } + if (`$selected` in row) result.$selected = row.$selected + return result as T + } + return view && typeof view === `object` ? { ...row, ...view } : row +} + /** Copy public containers while removing private route state at every depth. */ export function stripInternalRouteMetadata(value: unknown): unknown { return transformPublicContainers(value, (leaf) => leaf, INTERNAL_ROUTE_KEYS) diff --git a/packages/db/src/query/compiler/types.ts b/packages/db/src/query/compiler/types.ts index 3f2dddd6d2..f20a089bef 100644 --- a/packages/db/src/query/compiler/types.ts +++ b/packages/db/src/query/compiler/types.ts @@ -4,7 +4,10 @@ import type { CompilationResult } from './index.js' /** * Cache for compiled subqueries to avoid duplicate compilation */ -export type QueryCache = WeakMap> +export type QueryCache = WeakMap< + QueryIR, + [authored?: CompilationResult, source?: CompilationResult] +> /** * Lineage from optimized queries back to their user-defined queries. diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index ef59d73db6..2f9643da05 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -386,27 +386,44 @@ internal matches. The public-container copy matrix crosses reference-key type, ordered and unordered subqueries, materialization form, and parent/child updates. Reserved public row fields have a separate callback-input law (maintainer -decision, 2026-10-28). Direct source rows and structured whole-row source +decisions, 2026-10-28 and 2026-10-30). Direct source rows and structured whole-row source spreads expose genuine source `$key`, `$origin`, `$hasPendingWrites`, and `$collectionId`, including when a spread selects conflicting reserved names. A subquery projection that creates its own row shape, including a new functional object or functional spread, exposes none of those reserved fields -before downstream `fn.select()` code runs. An inline include value keeps its +before downstream `fn.select()`, `fn.where()`, or `fn.having()` code runs. +The October 30 decision extends the original selection-only promise to both +functional predicates. An inline include value keeps its authored selected shape. Sanitizing only the eventual Collection publication cannot satisfy this law, because user code may already have derived an ordinary field from a spoofed input. -The compiler enforces this source/projection boundary when a functional callback -consumes a QueryRef or joined subquery result, including inside a correlated -inline child. An inline-only QueryRef and the include's final authored value -retain their separate inline shape. The runtime -owner checks root, wrapped child and direct child callbacks at initial readiness, -held mutation, settlement and remote next use. Its bounded repair evidence is in +The compiler carries a callback view beside each authored selected-row +occurrence. FROM, joined-source and union adapters transport that view. +Functional callbacks read it without rewriting the query's authored row. +Private callback state never crosses the callback or result boundary. +An inline-only QueryRef and an inline child's final authored value retain +that authored shape, including when a predicate runs without a selection. +The selected-row normalization flag describes the output metadata view and +is independent of callback-view transport. A two-slot cache retains authored +and normalized compilations separately. Each ordered top-K retains its own +window; the source loader's registered plan covers every window that shares +that source, including subsequent moves. + +The runtime owner checks select, where and having source inputs through root, +wrapped child, direct child and real Effect receivers. Predicate-with-select +controls distinguish callback syntax from placement. Collection receivers +observe initial readiness, held mutation, settlement and remote next use. +Effects additionally observe every delivered value and previousValue, +exit/reentry and post-disposal silence. Its bounded evidence is in [the callback metadata record](../../../../../docs/contributing/oracle-reviews/callback-metadata.md). Reusing a compilation cache on one graph with the same input streams must retain the callback inputs and authored selected output that separate compilations produce. The exported `compileQuery` receiver checks both compilation orders -at initial graph drain, separately from the Collection lifecycle receivers. +through FROM, JOIN and union placement at initial graph drain. Its raw output +comparison is differential and supplies no independent Effect-root attribution +policy. Optimizer cloning can prevent an original raw-key cache hit in union +placement; that path checks propagation and transparency instead. Separate static receiving-type gaps remain in the [deferred oracle record](../../../../../docs/contributing/oracle-reviews/2026-10-29-pr-2171-deferred-oracles.md). diff --git a/packages/db/tests/query/ordered-compiler-cache-driver.ts b/packages/db/tests/query/ordered-compiler-cache-driver.ts new file mode 100644 index 0000000000..f5ef947cda --- /dev/null +++ b/packages/db/tests/query/ordered-compiler-cache-driver.ts @@ -0,0 +1,238 @@ +/** Controlled compiler/ordered-loader receiver for the ordered-work owner. + * One real on-demand Collection supplies a finite provider prefix. Two roots + * consume the exact authored QueryIR on one D2 graph, with shared or separate + * caches. Captures retain signed multiplicities; the owner supplies row truth. + * This is a core compiler/loader cut, not a public Collection window promise. + */ +import { D2, MultiSet, output } from '@tanstack/db-ivm' +import { createCollection } from '../../src/collection/index.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { Query, compileQuery, eq } from '../../src/query/index.js' +import { getQueryIR } from '../../src/query/builder/index.js' +import { QueryRef, UnionAll } from '../../src/query/ir.js' +import { OrderedSourceLoader } from '../../src/query/live/ordered-source-loader.js' +import { withOracleCleanup } from '../utils.js' +import type { CollectionSubscription } from '../../src/collection/subscription.js' +import type { OrderByOptimizationInfo } from '../../src/query/compiler/order-by.js' +import type { + QueryCache, + WindowOptions, +} from '../../src/query/compiler/types.js' +import type { QueryIR } from '../../src/query/ir.js' +import type { + LoadSubsetOptions, + LoadSubsetRequestResult, + SyncConfig, +} from '../../src/types.js' + +type Row = { id: number; rank: number } +export type OrderedCompilerCacheHistory = { + order: `authored-first` | `callback-first` + cache: `shared` | `separate` + path: `query-ref` | `joined-source` | `union-branch` + shape: `structured` | `functional` + direction: `asc` | `desc` + initial: { limit: number; offset: number } + moved: { limit: number; offset: number } +} +type Request = LoadSubsetOptions & { + onLoadSubsetResult?: ( + result: LoadSubsetRequestResult, + options: LoadSubsetOptions, + release: () => void, + ) => void +} +export async function receiveOrderedCompilerCacheHistory( + history: OrderedCompilerCacheHistory, + rows: ReadonlyArray, + check: ( + cut: `initial` | `moved` | `returned`, + actual: { authored: Array; callback: Array }, + ) => void, +) { + // The provider order is part of this controlled adapter's premise. This + // sorts provider data, not the expected answer; the owner computes that. + const provider = [...rows].sort((a, b) => + history.direction === `asc` ? a.rank - b.rank : b.rank - a.rank, + ) + let sync!: Parameters[`sync`]>[0] + const source = createCollection({ + id: `ordered-compiler-cache`, + getKey: (row: Row) => row.id, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: (params) => { + sync = params + sync.begin() + sync.write({ type: `insert`, value: { ...provider[0]! } }) + sync.commit() + sync.markReady() + return { loadSubset: () => true } + }, + }, + }) + let loader: OrderedSourceLoader | undefined + await withOracleCleanup(async () => { + const base = new Query() + .from({ chosen: source }) + .orderBy(({ chosen }) => chosen.rank, history.direction) + .offset(history.initial.offset) + .limit(history.initial.limit) + const selected = getQueryIR( + history.shape === `structured` + ? base.select(({ chosen }) => ({ + id: chosen.id, + rank: chosen.rank, + $key: `spoof`, + })) + : base.fn.select(({ chosen }) => ({ + id: chosen.id, + rank: chosen.rank, + $key: `spoof`, + })), + ) + const joined = getQueryIR( + new Query() + .from({ peer: source }) + .leftJoin( + { nested: new Query().from({ chosen: source }) }, + ({ peer, nested }) => eq(peer.id, nested.id), + ), + ) + joined.join![0]!.from = new QueryRef( + selected, + `nested`, + joined.join![0]!.from.bindingId, + ) + const consumed: QueryIR = + history.path === `joined-source` + ? { ...joined, fnSelect: ({ nested }) => nested } + : history.path === `union-branch` + ? { from: new UnionAll([selected]), fnSelect: (row) => row } + : { + from: new QueryRef(selected, `nested`), + fnSelect: ({ nested }) => nested, + } + const graph = new D2() + const input = graph.newInput<[number, Row]>() + const cache: QueryCache = new WeakMap() + const plans: Record = {} + let move!: (options: WindowOptions) => void + const compile = (query: QueryIR) => + compileQuery( + query, + { chosen: input, peer: input }, + { [source.id]: source }, + {}, + {}, + new Set(), + plans, + query === selected + ? (fn) => { + move = fn + } + : () => {}, + history.cache === `shared` ? cache : new WeakMap(), + ) + const [authored, callback] = + history.order === `authored-first` + ? [compile(selected), compile(consumed)] + : (() => { + const other = compile(consumed) + return [compile(selected), other] + })() + const authoredBag = new Map() + const callbackBag = new Map() + for (const [compiled, bag] of [ + [authored, authoredBag], + [callback, callbackBag], + ] as const) + compiled.pipeline.pipe( + output((message) => { + for (const [[, [value]], weight] of message.getInner()) { + if (value === undefined) continue // legal unmatched LEFT-join side + const id = (value as Row).id + bag.set(id, (bag.get(id) ?? 0) + weight) + } + }), + ) + graph.finalize() + input.sendData( + new MultiSet(source.toArray.map((row) => [[row.id, row as Row], 1])), + ) + graph.run() + let revision = 0 + const request = (options: Request, boundary: boolean) => { + if (!boundary) { + const page = provider.slice( + options.offset ?? 0, + (options.offset ?? 0) + (options.limit ?? provider.length), + ) + const fresh = page.filter((row) => !source.has(row.id)) + if (fresh.length) { + sync.begin() + for (const row of fresh) + sync.write({ type: `insert`, value: { ...row } }) + sync.commit() + revision++ + input.sendData( + new MultiSet( + fresh.map((row) => [[row.id, source.get(row.id)! as Row], 1]), + ), + ) + graph.run() + } + } + options.onLoadSubsetResult?.(true, options, () => {}) + } + const subscription = { + setOrderByIndex: () => {}, + readOrderedSnapshot: (options: LoadSubsetOptions) => + source.toArray + .sort((a, b) => + history.direction === `asc` ? a.rank - b.rank : b.rank - a.rank, + ) + .slice(0, options.limit ?? source.size) + .map((value) => ({ key: value.id, value })), + requestLimitedSnapshot: (options: Request) => request(options, false), + // All ranks are unique. A tie request adds no row beyond the installed + // boundary; it still settles its real loader continuation. + requestSnapshot: (options: Request) => request(options, true), + } + const info = Object.values(plans)[0]! + loader = new OrderedSourceLoader( + info, + subscription as unknown as CollectionSubscription, + `chosen`, + () => {}, + () => false, + () => revision, + ) + const snapshot = () => { + const expand = (bag: Map) => + [...bag].flatMap(([id, count]) => { + if (!Number.isSafeInteger(count) || count < 0) + throw new Error(`invalid ordered multiplicity ${id}:${count}`) + return Array.from({ length: count }, () => id) + }) + return { authored: expand(authoredBag), callback: expand(callbackBag) } + } + loader.start() + await loader.loadMore(0) + graph.run() + check(`initial`, snapshot()) + move(history.moved) + graph.run() + await loader.loadMore(1) + graph.run() + check(`moved`, snapshot()) + move(history.initial) + graph.run() + await loader.loadMore(2) + graph.run() + check(`returned`, snapshot()) + }, [() => loader?.dispose(), () => source.cleanup()]) +} diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index fd11aed199..20f2da83d4 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -1,5 +1,5 @@ import { fc, test as fcTest } from '@fast-check/vitest' -import { describe, expect, it, vi } from 'vitest' +import { afterAll, describe, expect, it, vi } from 'vitest' import { toArray } from '../../src/query/index.js' import { createCollection } from '../../src/collection/index.js' import { localOnlyCollectionOptions } from '../../src/local-only.js' @@ -20,6 +20,7 @@ import { evaluateReferenceExpression } from '../reference-expression-oracle.js' import { createOracleReach } from '../oracle-reach.js' import { flushPromises } from '../utils.js' import { withHistoryCleanup } from '../optimistic-history-oracle.js' +import { receiveOrderedCompilerCacheHistory } from './ordered-compiler-cache-driver.js' import { compareRows, expectDistinctRequests, @@ -7911,3 +7912,112 @@ declareOracle({ ], grammar: [exhaustiveScenarios, eagerFullJoinCase], }) + +/** A source's ordered acquisition must serve every compiled window that reads + * that lexical source. Authored/source metadata modes may have separate result + * streams; overwriting one window's demand with the other's cannot change the + * independent sorted slice. Authority: ordered acquisition and window laws in + * ARCHITECTURE.md. The exported compiler plus real loader is a controlled core + * receiving cut; the existing live Collection/Effect histories supply the host + * gate. These finite histories do not promise a universal operator-count bound. + */ +const compilerCacheWindows = ( + [`authored-first`, `callback-first`] as const +).flatMap((order) => + ([`shared`, `separate`] as const).flatMap((cache) => + ([`query-ref`, `joined-source`, `union-branch`] as const).flatMap((path) => + ([`structured`, `functional`] as const).flatMap((shape) => + ([`asc`, `desc`] as const).flatMap((direction) => + [ + { + initial: { limit: 1, offset: 0 }, + moved: { limit: 3, offset: 0 }, + }, + { + initial: { limit: 3, offset: 0 }, + moved: { limit: 1, offset: 0 }, + }, + { + initial: { limit: 1, offset: 0 }, + moved: { limit: 2, offset: 1 }, + }, + ].map((window) => ({ + order, + cache, + path, + shape, + direction, + ...window, + })), + ), + ), + ), + ), +) +const compilerWindowCuts = [`initial`, `moved`, `returned`] as const +const compilerWindowKey = (history: (typeof compilerCacheWindows)[number]) => + JSON.stringify(history) +const compilerWindowReach = createOracleReach( + `compiled ordered cache windows`, + compilerCacheWindows.flatMap((history) => + compilerWindowCuts.map((cut) => `${compilerWindowKey(history)}/${cut}`), + ), + { replaying: false }, +) +let compilerWindowEntered = false +afterAll(() => { + if (compilerWindowEntered) compilerWindowReach.assertReached() +}) +declareOracle({ + model: [expectedInstalledWindow], + grammar: [compilerCacheWindows], +}) +it.each(compilerCacheWindows)( + `serves every metadata-mode window of one ordered source: %j`, + async (history) => { + compilerWindowEntered = true + compilerWindowReach.beginHistory() + const rows = [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + { id: 3, rank: 3 }, + ] + const failures: Array = [] + const traces: Array = [] + await receiveOrderedCompilerCacheHistory(history, rows, (cut, actual) => { + const window = cut === `moved` ? history.moved : history.initial + const expected = { + authored: expectedInstalledWindow(rows, { + ...window, + direction: history.direction, + }), + callback: expectedInstalledWindow(rows, { + ...history.initial, + direction: history.direction, + }), + } + // Compare signed-bag occurrences, not D2 output delivery order. Rank order + // is independently exercised by the enclosing ordered-work owner. + const byId = (ids: Array) => [...ids].sort((a, b) => a - b) + try { + expect( + { authored: byId(actual.authored), callback: byId(actual.callback) }, + `${compilerWindowKey(history)}/${cut}`, + ).toStrictEqual({ + authored: byId(expected.authored), + callback: byId(expected.callback), + }) + } catch (error) { + failures.push(error) + traces.push({ cut, actual, expected }) + } + compilerWindowReach.reach(`${compilerWindowKey(history)}/${cut}`) + }) + if (failures.length) + throw new AggregateError( + failures, + `ordered compiler cache window mismatch: ${JSON.stringify(traces)}`, + { cause: failures[0] }, + ) + }, +) diff --git a/packages/db/tests/query/virtual-row-fields-input-driver.ts b/packages/db/tests/query/virtual-row-fields-input-driver.ts index 1352ef1c97..6f15a64e81 100644 --- a/packages/db/tests/query/virtual-row-fields-input-driver.ts +++ b/packages/db/tests/query/virtual-row-fields-input-driver.ts @@ -18,9 +18,14 @@ import { materialize, toArray, } from '../../src/query/index.js' +import { createEffect } from '../../src/query/effect.js' import { getQueryIR } from '../../src/query/builder/index.js' -import { QueryRef } from '../../src/query/ir.js' -import { mockSyncCollectionOptions, withOracleCleanup } from '../utils.js' +import { PropRef, QueryRef, UnionAll } from '../../src/query/ir.js' +import { + flushPromises, + mockSyncCollectionOptions, + withOracleCleanup, +} from '../utils.js' import { virtualFieldNames } from './virtual-row-fields-support.js' import { authoredMetadataCollision, @@ -35,7 +40,9 @@ import type { RefLeaf } from '../../src/query/builder/types.js' import type { QueryCache } from '../../src/query/compiler/types.js' import type { QueryIR } from '../../src/query/ir.js' import type { + MetadataCallbackKind, MetadataCallbackPath, + MetadataCompilerCachePath, MetadataInputDescription, MetadataInputRow, MetadataInputSnapshot, @@ -58,7 +65,13 @@ type ProjectionContext = { hasResult: true } export type CallbackInputCut = - `initial` | `held` | `settled` | `remote-next-use` + | `initial` + | `held` + | `settled` + | `remote-next-use` + | `exit` + | `reentry` + | `disposed` export type CallbackInputReport = { violations: Array<{ cut: CallbackInputCut @@ -215,7 +228,8 @@ export async function runMetadataCallbackHistory( description: MetadataInputDescription, path: MetadataCallbackPath | `direct-source`, reached: (cut: CallbackInputCut) => void, - receiver: `root` | `inline-child` | `inline-child-direct` = `root`, + receiver: `root` | `inline-child` | `inline-child-direct` | `effect` = `root`, + callbackKind: MetadataCallbackKind = `select`, ): Promise { const initial = freshRows() const union = path.startsWith(`union`) @@ -230,16 +244,24 @@ export async function runMetadataCallbackHistory( })) let cut: CallbackInputCut = `initial` let allowed = facts.map((fact) => expectedMetadataInput(fact, description)) + let enteredCallback = `select` const captures: Array<{ cut: CallbackInputCut + callback: string input: MetadataInputSnapshot }> = [] const project = (row: unknown, id: string | number) => { const observed = captureMetadataInput(row) - captures.push({ cut, input: observed }) + // A flat union HAVING callback also receives the compiler's selected-value + // context. It is not an ordinary field of the source value being judged. + // Capture reserved fields before removing that known context from ordinary + // fields; spreading first could hide inherited or nonenumerable metadata. + if (union && enteredCallback === `having`) + delete observed.ordinary.$selected + captures.push({ cut, callback: enteredCallback, input: observed }) return { id, observed } } - const makeQuery = (parentId?: RefLeaf) => { + const makeSelectQuery = (parentId?: RefLeaf) => { const input = projectInput(first, `chosen`, description) if (path === `direct-source`) { const direct = new Query().from({ direct: first }) @@ -306,8 +328,71 @@ export async function runMetadataCallbackHistory( : once.where(({ nested }) => eq(nested.id, parentId)) ).fn.select(({ nested }) => project(nested, nested.id)) } + const selects = callbackKind === `select` || callbackKind.endsWith(`-select`) + const makeQuery = (parentId?: RefLeaf) => { + const authored = makeSelectQuery(parentId) + if (callbackKind === `select`) return authored + const ir = getQueryIR(authored) + const recordSelection = ir.fnSelect! + const predicateKind = callbackKind.startsWith(`where`) ? `where` : `having` + const predicate = (row: Parameters[0]) => { + enteredCallback = predicateKind + try { + const derived = recordSelection(row) as Output + // A real predicate derives membership from the observed reserved field. + // It uses no expected answer or production provenance classification. + return derived.observed.fields.$key?.value !== `spoof-$key` + } finally { + enteredCallback = `select` + } + } + if (predicateKind === `where`) + ir.fnWhere = [...(ir.fnWhere ?? []), predicate] + else ir.fnHaving = [...(ir.fnHaving ?? []), predicate] + if (!selects) { + delete ir.fnSelect + if (!union) { + const alias = + path === `direct-source` + ? `direct` + : path === `nested-query-ref` + ? `outer` + : `nested` + ir.select = { + id: new PropRef([path === `joined-source` ? `peer` : alias, `id`]), + value: new PropRef([alias]), + } + } + } + // The builder receives its authored IR back; this changes callback syntax, + // not the receiver's production path or expected source facts. + return authored + } + const observeRows = (rows: ReadonlyArray): Array => + selects + ? outputRows(rows) + : rows.map((row) => { + const record = row as Record + const observed = captureMetadataInput(union ? row : record.value) + return { + id: record.id as string | number, + observed: receiver.startsWith(`inline-child`) + ? observed + : expectedMetadataRecord( + observed.present ? observed.ordinary : undefined, + ), + } + }) + const expectedOutput = (fact: MetadataSourceFact) => + selects + ? expectedMetadataInput(fact, description) + : receiver.startsWith(`inline-child`) + ? expectedMetadataInput(fact, description, true) + : expectedMetadataRecord( + expectedMetadataInput(fact, description, true).ordinary, + ) const query = - receiver === `root` + receiver === `root` || receiver === `effect` ? makeQuery() : new Query() .from({ parent: containingParents }) @@ -331,30 +416,117 @@ export async function runMetadataCallbackHistory( .where(({ result }) => eq(result.id, parent.id)), ), })) - const live = createLiveQueryCollection({ - query: () => query as unknown as QueryBuilder, - getKey: (row: Output) => row.id, - }) + // A real Effect receives the same model and callback grammar through both + // public channels. Its ledgers retain each value/previousValue before another + // action. No implicit Effect-root metadata policy is inferred here. + const effectRows = new Map() + const namedRows = new Map() + const effectErrors: Array = [] + const effectEvents: Array<{ + type: string + key: string | number + value: unknown + previousValue?: unknown + }> = [] + const namedEvents: typeof effectEvents = [] + const receive = ( + rows: typeof effectRows, + events: typeof effectEvents, + event: { + type: string + key: string | number + value: unknown + previousValue?: unknown + }, + ) => { + const captured = copyData(event) as (typeof effectEvents)[number] + events.push(captured) + const previous = rows.get(event.key) + if ( + event.type === `update` && + !isDeepStrictEqual(previous, event.previousValue) + ) + effectErrors.push({ + kind: `Effect previousValue differs from prior delivered value`, + previous, + event: captured, + }) + if (event.type === `enter` && previous !== undefined) + effectErrors.push({ + kind: `Effect enters an active occurrence`, + event: captured, + }) + if (event.type !== `enter` && previous === undefined) + effectErrors.push({ + kind: `Effect changes an absent occurrence`, + event: captured, + }) + if (event.type === `exit`) rows.delete(event.key) + else rows.set(event.key, copyData(event.value) as Output) + } + const effect = + receiver === `effect` + ? createEffect({ + query: () => query as unknown as QueryBuilder, + onBatch: (events) => { + for (const event of events) receive(effectRows, effectEvents, event) + }, + onEnter: (event) => receive(namedRows, namedEvents, event), + onUpdate: (event) => receive(namedRows, namedEvents, event), + onExit: (event) => receive(namedRows, namedEvents, event), + onError: (error) => { + effectErrors.push(error) + }, + onSourceError: (error) => { + effectErrors.push(error) + }, + }) + : undefined + const live = effect + ? undefined + : createLiveQueryCollection({ + query: () => query as unknown as QueryBuilder, + getKey: (row: Output) => row.id, + }) const report: CallbackInputReport = { violations: [], - callbackCounts: { initial: 0, held: 0, settled: 0, 'remote-next-use': 0 }, + callbackCounts: { + initial: 0, + held: 0, + settled: 0, + 'remote-next-use': 0, + exit: 0, + reentry: 0, + disposed: 0, + }, } - const check = () => { + let checkedEffectEvents = 0 + const check = async () => { + if (effect) await flushPromises() const premise = - live.isReady() && first.get(1)?.$hasPendingWrites === (cut === `held`) + (live?.isReady() ?? (!effect?.disposed && first.isReady())) && + first.get(1)?.$hasPendingWrites === (cut === `held`) if (!premise) report.violations.push({ cut, kind: `readiness or held-source premise missing`, actual: { - ready: live.isReady(), + ready: live?.isReady() ?? (!effect?.disposed && first.isReady()), pending: first.get(1)?.$hasPendingWrites, }, expected: { ready: true, pending: cut === `held` }, }) const current = captures.filter((capture) => capture.cut === cut) report.callbackCounts[cut] = current.length - if (current.length === 0) + const requiredCallback = callbackKind.startsWith(`where`) + ? `where` + : callbackKind.startsWith(`having`) + ? `having` + : `select` + const callbacksEntered = + current.some((capture) => capture.callback === requiredCallback) && + (!selects || current.some((capture) => capture.callback === `select`)) + if (!callbacksEntered) report.violations.push({ cut, kind: `callback not entered`, @@ -363,7 +535,7 @@ export async function runMetadataCallbackHistory( }) for (const capture of current) { const permitted = - cut === `initial` && path === `joined-source` + path === `joined-source` ? [...allowed, expectedMetadataRecord(undefined)] : allowed for (const kind of metadataInputViolations(capture.input, permitted)) @@ -376,26 +548,92 @@ export async function runMetadataCallbackHistory( } const expected = facts.map((fact) => ({ id: fact.row.id, - observed: expectedMetadataInput(fact, description), + observed: expectedOutput(fact), })) + // The unmatched LEFT-join peer remains a result after its source exits. + // The model's absent snapshot is judged at callbacks and output separately. + if (path === `joined-source` && cut === `exit`) + expected.push({ id: `1`, observed: expectedMetadataRecord(undefined) }) const raw = - receiver === `root` - ? live.toArray - : live.toArray.flatMap( - (row) => (row as unknown as { children: Array }).children, + receiver === `effect` + ? [...effectRows.values()] + : receiver === `root` + ? live!.toArray + : live!.toArray.flatMap( + (row) => (row as unknown as { children: Array }).children, + ) + if (effect) { + // Check each delivered value, not only the final active rows. Retraction + // evaluation can legally use before/after facts; it cannot deliver a + // snapshot outside those independently authored source states. + const permitted = allowed.flatMap((observed) => [ + { + id: observed.ordinary.id, + observed: selects + ? observed + : expectedMetadataRecord( + observed.present ? observed.ordinary : undefined, + ), + }, + ]) + if (path === `joined-source`) + for (const row of initial) + permitted.push({ + id: row.id, + observed: expectedMetadataRecord(undefined), + }) + for (const event of effectEvents.slice(checkedEffectEvents)) { + for (const value of [ + event.value, + ...(event.type === `update` ? [event.previousValue] : []), + ]) { + const actual = observeRows([value as object]) + if ( + !permitted.some( + (row) => + metadataOutputViolations(actual, [row as Output]).length === 0, + ) ) - const actual = outputRows(raw) + report.violations.push({ + cut, + kind: `Effect delivered value outside authored states`, + actual, + expected: permitted, + }) + } + } + checkedEffectEvents = effectEvents.length + } + if (effect) { + if (effectErrors.length || !isDeepStrictEqual(effectEvents, namedEvents)) + report.violations.push({ + cut, + kind: `Effect callback channels or event continuity`, + actual: { effectErrors, effectEvents, namedEvents }, + expected: `matching complete channels and continuous values`, + }) + for (const kind of metadataOutputViolations( + observeRows([...namedRows.values()]), + expected, + )) + report.violations.push({ + cut, + kind: `named Effect: ${kind}`, + actual: [...namedRows.values()], + expected, + }) + } + const actual = observeRows(raw) for (const kind of metadataOutputViolations(actual, expected)) report.violations.push({ cut, kind, actual, expected }) // Reach records the actual callback path and established source/output // boundary even when the law comparison is RED. It is not a passing count. - if (premise && current.length > 0 && raw.length === facts.length) - reached(cut) + if (premise && callbacksEntered) reached(cut) } let finish = async () => {} await withOracleCleanup(async () => { - await live.preload() - check() + if (live) await live.preload() + await check() cut = `held` const before = facts.map((fact) => expectedMetadataInput(fact, description)) facts = facts.map((fact) => @@ -419,7 +657,7 @@ export async function runMetadataCallbackHistory( await transaction.when(`settled`) finish = async () => {} } - check() + await check() cut = `settled` const pending = facts.map((fact) => expectedMetadataInput(fact, description), @@ -434,7 +672,7 @@ export async function runMetadataCallbackHistory( ...facts.map((fact) => expectedMetadataInput(fact, description)), ] await finish() - check() + await check() cut = `remote-next-use` const settled = facts.map((fact) => expectedMetadataInput(fact, description), @@ -452,12 +690,64 @@ export async function runMetadataCallbackHistory( receivingSource.utils.begin() receivingSource.utils.write({ type: `update`, value: { ...facts[1]!.row } }) receivingSource.utils.commit() - check() + await check() + if (effect) { + // Source exit/reentry exercises onExit and a fresh active interval. The + // same before/after authored facts judge both channels at each cut. + cut = `exit` + const departing = facts[1]! + allowed = facts.map((fact) => expectedMetadataInput(fact, description)) + facts = [facts[0]!] + const receiving = union ? second : first + // Use the actual branch owning the string key; no model source is moved. + receiving.utils.begin() + receiving.utils.write({ type: `delete`, value: { ...departing.row } }) + receiving.utils.commit() + await check() + cut = `reentry` + facts = [ + ...facts, + { ...departing, row: { ...departing.row, label: `string-reentry` } }, + ] + allowed = facts.map((fact) => expectedMetadataInput(fact, description)) + receiving.utils.begin() + receiving.utils.write({ type: `insert`, value: { ...facts[1]!.row } }) + receiving.utils.commit() + await check() + cut = `disposed` + await effect.dispose() + const beforeDisposal = { + captures: captures.length, + batch: effectEvents.length, + named: namedEvents.length, + } + receiving.utils.begin() + receiving.utils.write({ + type: `update`, + value: { ...facts[1]!.row, label: `after-disposal` }, + }) + receiving.utils.commit() + await flushPromises() + const afterDisposal = { + captures: captures.length, + batch: effectEvents.length, + named: namedEvents.length, + } + if (!effect.disposed || !isDeepStrictEqual(beforeDisposal, afterDisposal)) + report.violations.push({ + cut, + kind: `Effect delivery after disposal`, + actual: afterDisposal, + expected: beforeDisposal, + }) + else reached(cut) + } if (report.violations.length) throw new MetadataCallbackOracleFailure(report) }, [ () => finish(), - () => live.cleanup(), + () => effect?.dispose(), + () => live?.cleanup(), () => containingParents.cleanup(), () => peers.cleanup(), () => second.cleanup(), @@ -563,7 +853,15 @@ export async function runInlineMetadataCallbackHistory( }) const report: CallbackInputReport = { violations: [], - callbackCounts: { initial: 0, held: 0, settled: 0, 'remote-next-use': 0 }, + callbackCounts: { + initial: 0, + held: 0, + settled: 0, + 'remote-next-use': 0, + exit: 0, + reentry: 0, + disposed: 0, + }, } if (form === `collection`) { let unexpected: ReturnType | undefined @@ -702,12 +1000,13 @@ export async function runInlineMetadataCallbackHistory( * * Only the initial graph-drain cut is claimed here. Public source Collection * rows supply genuine metadata; independent authored facts predict callback - * input and both outputs. The raw selected output uses the same authored-shape - * judgment as an inline value, without claiming an actual include lifecycle. */ + * input and derived output. Raw selected output is compared across shared and + * separate caches; no implicit metadata law is assigned to an Effect root. */ export async function runMetadataCompilerCacheHistory( description: MetadataInputDescription, order: `authored-first` | `callback-first`, cache: `shared` | `separate`, + path: MetadataCompilerCachePath, reached: ( observation: `callback` | `derived-output` | `authored-output`, ) => void, @@ -721,74 +1020,136 @@ export async function runMetadataCompilerCacheHistory( })) const report: CallbackInputReport = { violations: [], - callbackCounts: { initial: 0, held: 0, settled: 0, 'remote-next-use': 0 }, + callbackCounts: { + initial: 0, + held: 0, + settled: 0, + 'remote-next-use': 0, + exit: 0, + reentry: 0, + disposed: 0, + }, } await withOracleCleanup(async () => { await first.preload() - const captures: Array = [] - const selected = getQueryIR(projectInput(first, `chosen`, description)) - const consumed: QueryIR = { - from: new QueryRef(selected, `nested`), - fnSelect: ({ nested }) => { - const observed = captureMetadataInput(nested) + const execute = (sharing: `shared` | `separate`) => { + const selected = getQueryIR(projectInput(first, `chosen`, description)) + const captures: Array = [] + const capture = (value: unknown, id?: string | number) => { + const observed = captureMetadataInput(value) captures.push(observed) - return { id: (nested as MetadataInputRow).id, observed } - }, - } - const graph = new D2() - const input = graph.newInput<[string | number, MetadataInputRow]>() - const sharedCache: QueryCache = new WeakMap() - const compile = (query: QueryIR) => - compileQuery( - query, - { chosen: input }, - { [first.id]: first }, - {}, - {}, - new Set(), - {}, - () => {}, - cache === `shared` ? sharedCache : new WeakMap(), + return { id: id ?? (value as MetadataInputRow).id, observed } + } + const joined = getQueryIR( + new Query() + .from({ peer: first }) + .leftJoin( + { nested: projectInput(first, `chosen`, description) }, + ({ peer, nested }) => eq(peer.id, nested.id), + ), ) - const [authored, callback] = - order === `authored-first` - ? [compile(selected), compile(consumed)] - : (() => { - const compiledCallback = compile(consumed) - return [compile(selected), compiledCallback] - })() - const authoredDeltas: Array<{ row: Output; weight: number }> = [] - const derivedDeltas: Array<{ row: Output; weight: number }> = [] - authored.pipeline.pipe( - output((message) => { - for (const [[, [value]], weight] of message.getInner()) - authoredDeltas.push({ - row: { - id: (value as MetadataInputRow).id, - observed: captureMetadataInput(value), - }, - weight, - }) - }), - ) - callback.pipeline.pipe( - output((message) => { - for (const [[, [value]], weight] of message.getInner()) { - const row = value as Output - derivedDeltas.push({ - row: { - id: row.id, - // Copy the selected ordinary capture at output delivery too. - observed: copyData(row.observed) as MetadataInputSnapshot, - }, - weight, - }) - } - }), + // Install the exact shared IR after authoring the legal join expression. + // Query builders normally clone placements, which concealed the cache bug. + joined.join![0]!.from = new QueryRef( + selected, + `nested`, + joined.join![0]!.from.bindingId, + ) + const consumed: QueryIR = + path === `joined-source` + ? { + ...joined, + fnSelect: ({ nested, peer }) => + capture(nested, (peer as MetadataInputRow).id), + } + : path === `union-branch` + ? { + from: new UnionAll([selected]), + fnSelect: (value) => capture(value), + } + : { + from: new QueryRef(selected, `nested`), + fnSelect: ({ nested }) => capture(nested), + } + const graph = new D2() + const input = graph.newInput<[string | number, MetadataInputRow]>() + const sharedCache: QueryCache = new WeakMap() + const readCache = sharedCache.get.bind(sharedCache) + let exactQueryReads = 0 + sharedCache.get = (query) => { + if (query === selected) exactQueryReads++ + return readCache(query) + } + const compile = (query: QueryIR) => + compileQuery( + query, + { chosen: input, peer: input }, + { [first.id]: first }, + {}, + {}, + new Set(), + {}, + () => {}, + sharing === `shared` ? sharedCache : new WeakMap(), + ) + const [authored, callback] = + order === `authored-first` + ? [compile(selected), compile(consumed)] + : (() => { + const compiledCallback = compile(consumed) + return [compile(selected), compiledCallback] + })() + const authoredDeltas: Array<{ row: Output; weight: number }> = [] + const derivedDeltas: Array<{ row: Output; weight: number }> = [] + authored.pipeline.pipe( + output((message) => { + for (const [[, [value]], weight] of message.getInner()) + authoredDeltas.push({ + row: { + id: (value as MetadataInputRow).id, + observed: captureMetadataInput(value), + }, + weight, + }) + }), + ) + callback.pipeline.pipe( + output((message) => { + for (const [[, [value]], weight] of message.getInner()) { + const row = value as Output + derivedDeltas.push({ + row: { + id: row.id, + // Copy the selected ordinary capture at output delivery too. + observed: copyData(row.observed) as MetadataInputSnapshot, + }, + weight, + }) + } + }), + ) + graph.finalize() + input.sendData( + new MultiSet(first.toArray.map((row) => [[row.id, row], 1])), + ) + graph.run() + + return { captures, authoredDeltas, derivedDeltas, exactQueryReads } + } + const { captures, authoredDeltas, derivedDeltas, exactQueryReads } = + execute(cache) + const reference = execute(`separate`) + if ( + process.env.TANSTACK_DB_ORACLE_STATISTICS === `1` && + description.shape === `structured-projection` && + description.collision === `none` && + cache === `shared` ) - graph.finalize() - input.sendData(new MultiSet(first.toArray.map((row) => [[row.id, row], 1]))) - graph.run() + console.info(`compiler exact raw-query reads`, { + path, + order, + reads: exactQueryReads, + }) // Independent signed-bag bookkeeping preserves duplicate contributions and // retractions. It does not replace the source/projection judgment. @@ -828,10 +1189,35 @@ export async function runMetadataCompilerCacheHistory( id: fact.row.id, observed: expectedInput[index]!, })) - const expectedAuthored = facts.map((fact) => ({ - id: fact.row.id, - observed: expectedMetadataInput(fact, description, true), - })) + // Cache transparency is a relation, not an Effect-root attribution rule. + // Separate compilation supplies the other execution, never the O12 truth. + // The independent source/projection model still judges callback inputs and + // complete ordinary derived outputs in both executions below. + const expectedAuthored = accumulate(reference.authoredDeltas) + const referenceDerived = accumulate(reference.derivedDeltas) + for (const capture of reference.captures) + for (const kind of metadataInputViolations( + capture, + path === `joined-source` + ? [...expectedInput, expectedMetadataRecord(undefined)] + : expectedInput, + )) + report.violations.push({ + cut: `initial`, + kind: `reference: ${kind}`, + actual: capture, + expected: expectedInput, + }) + for (const kind of metadataOutputViolations( + referenceDerived, + expectedDerived, + )) + report.violations.push({ + cut: `initial`, + kind: `reference: ${kind}`, + actual: referenceDerived, + expected: expectedDerived, + }) report.callbackCounts.initial = captures.length if (!captures.length) report.violations.push({ @@ -841,7 +1227,12 @@ export async function runMetadataCompilerCacheHistory( expected: `one or more`, }) for (const capture of captures) - for (const kind of metadataInputViolations(capture, expectedInput)) + for (const kind of metadataInputViolations( + capture, + path === `joined-source` + ? [...expectedInput, expectedMetadataRecord(undefined)] + : expectedInput, + )) report.violations.push({ cut: `initial`, kind, diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts index bf363cb492..06358ec70e 100644 --- a/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.test.ts @@ -2,11 +2,16 @@ * # Reserved fields must be correct before user JavaScript consumes them. * * The shared input oracle states the maintainer's source/projection/inline law. - * This finite receiver exhausts 231 root histories, 231 wrapped child histories - * and 165 direct child histories. Another 198 inline-value/rejection histories - * and three direct-source controls exercise the neighboring boundaries. - * Another 132 histories use the exported compiler with exact shared QueryIR, - * both compilation orders and shared/separate caches at initial graph drain. + * This finite receiver exhausts 1,155 root histories, 1,155 wrapped child + * histories, 825 direct child histories and 1,155 real Effect histories. + * Each crosses select, where, having and predicate-with-select controls. + * Another 198 inline value/rejection histories and 20 direct-source controls exercise neighboring + * boundaries. Effects also check exit, reentry, delivery continuity in both + * callback channels, every delivered value/previousValue, and post-disposal + * silence. Another 396 histories use the exported compiler across FROM, JOIN + * and union placement, both compilation orders and shared/separate caches at + * initial graph drain. Optimizer cloning can prevent a placement from reusing + * the original raw key; explicit counters distinguish that from cache reuse. * It invokes real query callbacks and compares their copied input and ordinary derived output at * initial readiness, held mutation, settlement and later remote use. The * materializer receiver uses the same judgment, not a competing child model. @@ -32,6 +37,7 @@ import { expectedMetadataRecord, expectedSourceVirtualFields, metadataCallbackHistories, + metadataCallbackKinds, metadataCallbackReceiverHistories, metadataCompilerCacheHistories, metadataInlineHistories, @@ -47,6 +53,8 @@ import { import type { MetadataSourceFact } from './virtual-row-fields-input-oracle.js' const cuts = [`initial`, `held`, `settled`, `remote-next-use`] as const +const receiverCuts = (receiver: string) => + receiver === `effect` ? [...cuts, `exit`, `reentry`, `disposed`] : cuts const key = (history: { shape: string collision: string @@ -56,10 +64,11 @@ const key = (history: { // Escape only the display label; the grammar and reach retain exact names. const callbackCases = metadataCallbackReceiverHistories.map((history) => ({ ...history, - label: `${history.receiver}/${history.path}/${key(history)}`.replaceAll( - `$`, - `field:`, - ), + label: + `${history.kind}/${history.receiver}/${history.path}/${key(history)}`.replaceAll( + `$`, + `field:`, + ), })) const inlineCases = metadataInlineHistories.map((history) => ({ ...history, @@ -70,10 +79,11 @@ const inlineCases = metadataInlineHistories.map((history) => ({ })) const compilerCases = metadataCompilerCacheHistories.map((history) => ({ ...history, - label: `${history.order}/${history.cache}/${key(history)}`.replaceAll( - `$`, - `field:`, - ), + label: + `${history.path}/${history.order}/${history.cache}/${key(history)}`.replaceAll( + `$`, + `field:`, + ), })) const compilerObservations = [ `callback`, @@ -85,7 +95,7 @@ const compilerReach = createOracleReach( metadataCompilerCacheHistories.flatMap((history) => compilerObservations.map( (observation) => - `${history.order}/${history.cache}/${key(history)}/${observation}`, + `${history.path}/${history.order}/${history.cache}/${key(history)}/${observation}`, ), ), { replaying: false }, @@ -94,12 +104,18 @@ const callbackReach = createOracleReach( `functional metadata callback`, [ ...metadataCallbackReceiverHistories.flatMap((history) => - cuts.map( - (cut) => `${history.receiver}/${history.path}/${key(history)}/${cut}`, + receiverCuts(history.receiver).map( + (cut) => + `${history.kind}/${history.receiver}/${history.path}/${key(history)}/${cut}`, ), ), - ...[`root`, `inline-child`, `inline-child-direct`].flatMap((receiver) => - cuts.map((cut) => `${receiver}/direct-source/${cut}`), + ...metadataCallbackKinds.flatMap((kind) => + [`root`, `inline-child`, `inline-child-direct`, `effect`].flatMap( + (receiver) => + receiverCuts(receiver).map( + (cut) => `${kind}/${receiver}/direct-source/${cut}`, + ), + ), ), ], { replaying: false }, @@ -142,6 +158,7 @@ declareOracle({ model: [expectedMetadataInput], grammar: [ metadataCallbackHistories, + metadataCallbackKinds, metadataCallbackReceiverHistories, metadataInlineHistories, metadataCompilerCacheHistories, @@ -158,9 +175,10 @@ test.each(callbackCases)( history.path, (cut) => callbackReach.reach( - `${history.receiver}/${history.path}/${key(history)}/${cut}`, + `${history.kind}/${history.receiver}/${history.path}/${key(history)}/${cut}`, ), history.receiver, + history.kind, ) expect( report.violations, @@ -168,9 +186,15 @@ test.each(callbackCases)( ).toStrictEqual([]) }, ) -test.each([`root`, `inline-child`, `inline-child-direct`] as const)( - `receives direct source metadata before deriving ordinary values: %s`, - async (receiver) => { +test.each( + metadataCallbackKinds.flatMap((kind) => + ([`root`, `inline-child`, `inline-child-direct`, `effect`] as const).map( + (receiver) => ({ kind, receiver }), + ), + ), +)( + `receives direct source metadata before deriving ordinary values: %j`, + async ({ kind, receiver }) => { callbackEntered = true callbackReach.beginHistory() const report = await runMetadataCallbackHistory( @@ -180,8 +204,9 @@ test.each([`root`, `inline-child`, `inline-child-direct`] as const)( authoredPending: false, }, `direct-source`, - (cut) => callbackReach.reach(`${receiver}/direct-source/${cut}`), + (cut) => callbackReach.reach(`${kind}/${receiver}/direct-source/${cut}`), receiver, + kind, ) expect(report.violations).toStrictEqual([]) }, @@ -208,8 +233,9 @@ test.each(inlineCases)( ) // The initial compiler graph-drain cut is distinct from the Collection -// lifecycle cuts above. Both compilation orders must retain authored output -// while source/projection metadata is judged before downstream JavaScript. +// lifecycle cuts above. Both orders must preserve the cache-transparency +// relation. Independent source/projection judgments still check both executions; +// raw output does not borrow an implicit Effect-root attribution policy. test.each(compilerCases)( `preserves metadata across compiler cache modes: $label`, async (history) => { @@ -219,9 +245,10 @@ test.each(compilerCases)( history, history.order, history.cache, + history.path, (observation) => compilerReach.reach( - `${history.order}/${history.cache}/${key(history)}/${observation}`, + `${history.path}/${history.order}/${history.cache}/${key(history)}/${observation}`, ), ) expect( diff --git a/packages/db/tests/query/virtual-row-fields-input-oracle.ts b/packages/db/tests/query/virtual-row-fields-input-oracle.ts index 28de6a76e9..f64b6916be 100644 --- a/packages/db/tests/query/virtual-row-fields-input-oracle.ts +++ b/packages/db/tests/query/virtual-row-fields-input-oracle.ts @@ -2,7 +2,8 @@ * # What may user JavaScript learn from a query input? * * Authority: the maintainer's O12 decision, recorded in the RC challenge on - * 2026-10-28, and ARCHITECTURE.md's functional-input boundary. Direct source + * 2026-10-28, extended by the maintainer on 2026-10-30 to fn.select, fn.where + * and fn.having, and ARCHITECTURE.md's functional-input boundary. Direct source * returns and structured source spreads expose genuine source metadata. * Own-shape projections (including functional spreads) expose no reserved * fields before a downstream callback. Inline include values keep authored @@ -65,9 +66,17 @@ export const metadataCallbackHistories = metadataInputDescriptions.flatMap( (description) => metadataCallbackPaths.map((path) => ({ ...description, path })), ) +export const metadataCallbackKinds = [ + `select`, + `where`, + `having`, + `where-select`, + `having-select`, +] as const +export type MetadataCallbackKind = (typeof metadataCallbackKinds)[number] export const metadataCallbackReceiverHistories = metadataCallbackHistories.flatMap((history) => - ([`root`, `inline-child`, `inline-child-direct`] as const) + ([`root`, `inline-child`, `inline-child-direct`, `effect`] as const) // A flat union has no named source reference for include correlation. // Admission requires the QueryRef used by union-query-ref or the wrapped // receiver; direct union-first/last are not legal include children. @@ -76,7 +85,9 @@ export const metadataCallbackReceiverHistories = receiver !== `inline-child-direct` || (history.path !== `union-first` && history.path !== `union-last`), ) - .map((receiver) => ({ ...history, receiver })), + .flatMap((receiver) => + metadataCallbackKinds.map((kind) => ({ ...history, receiver, kind })), + ), ) export const metadataInlineHistories = metadataInputDescriptions.flatMap( (description) => @@ -92,14 +103,24 @@ export const metadataInlineHistories = metadataInputDescriptions.flatMap( // Reusing one selected query across two roots must preserve both its authored // output and the source/projection judgment before a downstream callback. // Separate caches distinguish that promise from an ordinary projection defect. +export const metadataCompilerCachePaths = [ + `query-ref`, + `joined-source`, + `union-branch`, +] as const +export type MetadataCompilerCachePath = + (typeof metadataCompilerCachePaths)[number] export const metadataCompilerCacheHistories = metadataInputDescriptions.flatMap( (description) => ([`authored-first`, `callback-first`] as const).flatMap((order) => - ([`shared`, `separate`] as const).map((cache) => ({ - ...description, - order, - cache, - })), + ([`shared`, `separate`] as const).flatMap((cache) => + metadataCompilerCachePaths.map((path) => ({ + ...description, + order, + cache, + path, + })), + ), ), ) export type MetadataInputSnapshot = { From 86510968691e0080a7484881ed7f54ba58f110af Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 14:49:23 -0600 Subject: [PATCH 07/19] test(db): inspect compiler result data slots --- packages/db/tests/query/compiler/basic.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/db/tests/query/compiler/basic.test.ts b/packages/db/tests/query/compiler/basic.test.ts index f3209ba94a..53aa58a9d6 100644 --- a/packages/db/tests/query/compiler/basic.test.ts +++ b/packages/db/tests/query/compiler/basic.test.ts @@ -104,8 +104,12 @@ describe(`Query2 Compiler`, () => { const collection = messages[0]! expect(collection.getInner()).toHaveLength(4) - // Check the structure of the results - should be the raw user objects in tuple format - const results = collection.getInner().map(([data]) => data) + // Check the row and ordering slots; later slots carry compiler context. + const results = collection + .getInner() + .map( + ([[key, [row, orderByIndex]]]) => [key, [row, orderByIndex]] as const, + ) expect(results).toContainEqual([1, [sampleUsers[0], undefined]]) expect(results).toContainEqual([2, [sampleUsers[1], undefined]]) expect(results).toContainEqual([3, [sampleUsers[2], undefined]]) @@ -154,8 +158,12 @@ describe(`Query2 Compiler`, () => { graph.run() - // Check the structure of the results - const results = messages[0]!.getInner().map(([data]) => data) + // Check the row and ordering slots; later slots carry compiler context. + const results = messages[0]! + .getInner() + .map( + ([[key, [row, orderByIndex]]]) => [key, [row, orderByIndex]] as const, + ) expect(results).toContainEqual([ 1, From 8464ec97725a14c9df103e170d9367737c9ef41a Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 15:08:55 -0600 Subject: [PATCH 08/19] test(db): scope replay error order to subscriptions --- docs/contributing/oracle-coverage.md | 17 +++ .../oracle-reviews/callback-metadata.md | 95 ++++++++++++++++ ...ad-subset-replay-refinement-oracle.test.ts | 104 +++++++++++++++++- 3 files changed, 214 insertions(+), 2 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index b43c65d5d7..53b5e8d77d 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -1147,6 +1147,23 @@ that fails one receiver's current request while comparing another receiver's status at the same checkpoint. This is a stated observation limit, not a retirement of the per-demand error-ownership law. +The replay error-identity comparison keeps subscription ownership separate +from the source-wide publication barrier. CI seed `1926301329`, path +`251:2:1:4:2:2`, exposed a false failure: an include replay rejected while a +join replay still loaded, and the comparison required the join's earlier +failure instead. Adapter request data now identifies each load's subscription. +Only failures and covering replacements within that subscription compete for +its replay error. A 96-history finite grammar crosses root/marker replay, +oldest/newest failure order, four join forms, both directions and three window +widths. It measures 52 histories with multiple failures of one subscription +and six with sibling failures; both require at least five. The old source-wide +comparison and a production mutant that returns the last replay failure each +fail the intended identity assertion. Publication, request-error, retirement, +readiness and convergence comparisons and campaign budgets remain. This is a +controlled Collection/live-query witness, with no provider or framework claim. +Evidence and the oracle self-review are in +[the callback metadata record](oracle-reviews/callback-metadata.md#replay-error-identity-ci-follow-up-2026-10-30). + The ordered replay owner distinguishes replay renewal from an ordinary lazy demand created during another subscription's replay. Its adapter records the leases held at truncate, then matches subscription identity and independently diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 44fec39a80..2b77de43c9 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -601,3 +601,98 @@ the report hash itself refer to its own contents: | `packages/db/tests/query/virtual-row-fields-input-oracle.test.ts` | `58ae2e54947145220a8b78a28cadc37456e3d11d507ce36846843f002d79550d` | | `packages/db/tests/query/ordered-compiler-cache-driver.ts` | `ba7b244250754d9a5d8480847562c26907edf192c8a028973dca0de32e6c43e6` | | `packages/db/tests/query/ordered-work-oracle.property.test.ts` | `a8e2f5884570f0659764ad0b7be7398c432b411fcf796dd007ddd3f4bb9d5e68` | + + +## Replay error identity CI follow-up (2026-10-30) + +The parent is `86510968691e0080a7484881ed7f54ba58f110af`. +This follow-up repairs an oracle scope error, with no production changes. +The earlier production and metadata-oracle fingerprints above remain intact. + +**Law and authority.** ARCHITECTURE.md, “Atomic window publication” and +“Replay participants and failure”, assign each subscription its replay's +normalized Error, the earliest failure that still decides it. Subscriptions +share a publication barrier. Their failures do not acquire a common error +order: one replay can fail while a sibling still loads. + +**Experiment.** CI failed `replay.ordered-failed-replay` with seed +`1926301329`, path `251:2:1:4:2:2`. Exact replay on unchanged production +failed the identity comparison. The trace holds loads, truncates markers, +requests window 0/1, then fails newest and oldest held loads. It uses an +ascending anti-joined window with inline includes and on-demand markers. +Adapter diagnostics identify failure load 10 with subscription 1 and the +move's rejected load 8 with subscription 2; load 9 of subscription 1 is still +pending. The source-wide comparison incorrectly made load 10 decide the +other subscription's error. This is evidence against the comparison, not a +production error or a new law decision. + +The corrected model records subscription identity from adapter request data. +Competing failures and covering replacements belong to the same subscription. +The source-wide retention model stays shared. The exact replay passes. +A 96-history finite grammar crosses two directions, four join forms, three +window widths, root/marker replay and oldest/newest failure order. Each uses +the existing real Collection/live-query driver and every shared comparison. +Declared reach records 52 histories with multiple own-subscription failures +and six with sibling failures, each with minimum five. The normal package +owner run passes all 43 tests, including unchanged fixed/random budgets. +Type checking reports no errors. + +**Calibration.** Restoring the source-wide error order fails the finite grammar +at the same marker Error identity assertion. A temporary production mutant +that returns the last replay failure instead of the first fails its root +Error identity assertion. Both reach their intended comparisons, with no type +errors. The original production source is restored byte-for-byte. +These controls distinguish an ownership correction from an unconditional +waiver of replay error order. + +**Inference and limits.** The repair preserves the accepted per-subscription +error law and freedom between sibling replay verdicts. Existing retention, +request-error, demand-retirement, readiness, window and convergence checks and +all campaign budgets remain. This bounds evidence to one query, one-level +includes, the existing finite source domain and a controlled adapter in the +real Collection/live-query path. Existing peer exclusions and receiving-provider +and framework limits remain; this is not universal replay closure. + +The evidence directory is +`/private/tmp/callback-metadata/ci-monitor/20261030T205752Z`: +`replay-red.*`, `replay-diagnostic.log`, `replay-scope-green.*`, +`replay-matrix-green.*`, `replay-package-green.*`, `source-wide-check.*`, +`last-replay-error.*`, and `replay-control-summary.json`. +The exact replay command is: + +```sh +TANSTACK_DB_ORACLE_SEED=1926301329 \ +TANSTACK_DB_ORACLE_PATH=251:2:1:4:2:2 \ +TANSTACK_DB_ORACLE_PROPERTY=replay.ordered-failed-replay \ +pnpm_config_verify_deps_before_run=false pnpm --filter @tanstack/db exec \ + vitest run tests/query/load-subset-replay-refinement-oracle.test.ts \ + -t 'keeps the last publication through a failed replay' \ + --coverage.enabled=false --maxWorkers=1 +``` + +This is an oracle self-review, not independent review. ORC conformance: + +| Requirement | Outcome | +| --- | --- | +| 001, 002 | Existing architecture authority; identity from adapter input, no production verdict helper computes expectations. | +| 003 | Contract, model distinction, finite grammar and observation remain explained beside executable code. | +| 004 | Matrix reconstructs the CI trace; removing sibling ownership reproduces the false failure. Finite domains are stated; failure of a load outside a waited replay remains outside the identity premise. | +| 005 | Real Collections and `setWindow` rejection at replay settlement; no missing source-fact hypothesis is needed for this error-identity comparison. | +| 006 | Source-wide comparison control and last-error production mutant fail intended assertions. | +| 007 | Existing fixed/random properties, direct seed/path replay and budgets remain unchanged; finite matrix is bounded enumeration. | +| 008, 009 | Subscription identity distinguishes a sibling failure from one's own; both can share the publication barrier. Mapping is beside the model. | +| 010 | Original and diagnostic traces retained; existing finally cleanup and failure propagation remain. | +| 011 | Shared production/model scope fault was disproved by independent adapter identities and the architecture rule; a second runtime model is unnecessary for this comparison correction. | +| 012 | Parent and source binding, evidence chronology and all applicable outcomes recorded here. | +| 013 | Sibling and same-subscription neighbors distinguish global error order from per-replay order. | +| 014 | Controlled adapter claim only; no real-provider or framework transfer claimed. | +| 015 | Both declared states exceed minimum five, with 52 and six histories respectively. | +| 016 | Matrix supplies histories to the same driver and comparison, with no separate expected error. | + +Final SHA-256 binding (the report itself is excluded): + +| Source | SHA-256 | +| --- | --- | +| `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts` | `119f75c35a0b5c472d211997a9f677fa0a92e605d99c8bde46aeda8c14a04560` | +| `packages/db/src/collection/subscription.ts` | `315a043966bf03ae7671e1e34338692d93eb91f04223a7c079a7585c4a54860b` | +| `packages/db/src/query/live/collection-config-builder.ts` | `0c00f76f05ecdea1c842e47e7749ed5dafbbb32f5c9f8a0937faf2776b4f474d` | diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts index 8a11da8ce3..248bae88c3 100644 --- a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts @@ -165,7 +165,8 @@ import type { LoadSubsetOptions, SyncConfig } from '../../src/types.js' * node may list a failure, and the replay-error identity law skips peer * histories. The replay-error identity law judges only rejections with a failure of a replay * the move waited on; a move that resumes into another failed replay is not - * judged. + * judged. Each subscription owns its replay verdict, even when publication + * is shared with a sibling. Error-order comparisons keep that distinction. */ type OrderedFailureState = `initiating-keyed-request-failed` | `keyed-reacquisition-failed` @@ -316,6 +317,8 @@ type OrderedReach = { // Deferred window moves that rejected with a failure of a replay they // waited on. deferredMoveRejections: number + ownReplayFailureComparisons: number + siblingReplayFailureComparisons: number replayLoads: number abortRejections: number preloadChecks: number @@ -463,6 +466,8 @@ async function checkOrderedReplay( error?: Error // Its source, the keys it asked for, and whether it is an ordered read. source?: BarrierSource + // Publication is shared, but each subscription owns its replay verdict. + subscription: LoadSubsetOptions[`subscription`] keys?: Set ordered?: boolean } @@ -890,6 +895,8 @@ async function checkOrderedReplay( requestErrorChecks: 0, successWhileSubscriptionOwes: 0, deferredMoveRejections: 0, + ownReplayFailureComparisons: 0, + siblingReplayFailureComparisons: 0, replayLoads: 0, abortRejections: 0, preloadChecks: 0, @@ -942,6 +949,7 @@ async function checkOrderedReplay( // owner, whose release unloads it. acquired.set(options, (acquired.get(options) ?? 0) + 1) const load: Load = { + subscription: options.subscription, epoch: truncates, outcome: `failed`, source: `markers`, @@ -990,6 +998,7 @@ async function checkOrderedReplay( const startedReady = allReady() acquired.set(options, (acquired.get(options) ?? 0) + 1) const load: Load = { + subscription: options.subscription, epoch: truncates, outcome: `pending`, source: `markers`, @@ -1146,6 +1155,7 @@ async function checkOrderedReplay( const startedReady = allReady() acquired.set(options, (acquired.get(options) ?? 0) + 1) const load: Load = { + subscription: options.subscription, epoch: truncates, outcome: `pending`, source: `rows`, @@ -1367,7 +1377,12 @@ async function checkOrderedReplay( // failing, as a demand's replacement does, no longer count. A failure of a // load in no waited replay is the move's own window failure after it // resumed, which this law does not judge. + // The source-wide barrier combines subscriptions only for publication. + // Their verdicts remain separate: a sibling can fail while another replay + // still loads. Only failures of the rejecting subscription compete for its + // earliest error. Subscription identity comes from adapter request data. const covers = (later: Load, earlier: Load) => + later.subscription === earlier.subscription && later.ordered === earlier.ordered && later.keys !== undefined && earlier.keys !== undefined && @@ -1392,13 +1407,32 @@ async function checkOrderedReplay( ) if (candidates.length === 0) return reach.deferredMoveRejections++ + const rejected = loads.find((load) => load.error === error)! + for (const barrier of candidates) { + if ( + barrier.history.filter( + (load) => load.subscription === rejected.subscription, + ).length > 1 + ) + reach.ownReplayFailureComparisons++ + if ( + barrier.history.some( + (load) => load.subscription !== rejected.subscription, + ) + ) + reach.siblingReplayFailureComparisons++ + } const earliest = candidates.some((barrier) => barrier.history .slice( 0, barrier.history.findIndex((load) => load.error === error), ) - .every((earlier, at) => !stillDecides(barrier, earlier, at)), + .every( + (earlier, at) => + earlier.subscription !== rejected.subscription || + !stillDecides(barrier, earlier, at), + ), ) if (!earliest) moveErrorViolation ??= `move ${move} rejected with "${error.message}", not its replay's earliest failure that still decides it` @@ -2429,6 +2463,68 @@ describe(`Ordered windows across truncate replays`, () => { ) }) + // This finite history grammar varies which source replays, the order of + // two load failures, join form, direction and window width. Join plus include + // gives sibling subscriptions; the root replay gives same-subscription + // failures. The same comparison checks both, without an expected error + // copied from production or a source-wide error order. + it(`keeps replay error order within each subscription`, async () => { + const declared = createOracleReach( + `replay.subscription-error-order`, + [ + `ownReplayFailureComparisons`, + `siblingReplayFailureComparisons`, + ] as const, + { minimum: 5 }, + ) + for (const direction of [`asc`, `desc`] as const) { + for (const joined of [false, `some`, `none`, `inner`] as const) { + for (const limit of [1, 2, 3] as const) { + for (const markers of [false, true]) { + for (const newest of [false, true]) { + declared.beginHistory() + const observed = await checkOrderedReplay( + [ + { type: `hold` }, + markers + ? { type: `truncateMarkers` } + : { type: `truncate`, armAfterStart: false }, + { type: `window`, offset: 0, limit }, + { type: `settle`, ok: false, newest }, + { + type: `settle`, + ok: false, + newest: !newest, + keepHolding: false, + }, + ], + { + ...ordinaryOrderedScenario, + direction, + joined, + limit, + markerMode: `on-demand`, + includes: true, + }, + ) + for (const state of [ + `ownReplayFailureComparisons`, + `siblingReplayFailureComparisons`, + ] as const) { + if (observed[state] > 0) declared.reach(state) + } + } + } + } + } + } + console.info( + `replay.subscription-error-order`, + Object.fromEntries(declared.counts()), + ) + declared.assertReached() + }, 120_000) + it.each(orderedReplayCampaigns)( `keeps the ordered window under load and release faults (%s campaign)`, async (campaign) => { @@ -2455,6 +2551,8 @@ describe(`Ordered windows across truncate replays`, () => { requestErrorChecks: 0, successWhileSubscriptionOwes: 0, deferredMoveRejections: 0, + ownReplayFailureComparisons: 0, + siblingReplayFailureComparisons: 0, replayLoads: 0, abortRejections: 0, preloadChecks: 0, @@ -2493,6 +2591,8 @@ describe(`Ordered windows across truncate replays`, () => { `discardedReplayMoves`, `requestErrorChecks`, `deferredMoveRejections`, + `ownReplayFailureComparisons`, + `siblingReplayFailureComparisons`, `successWhileSubscriptionOwes`, ]) // Counts are histories, rather than repeated events in one trace. From c33e18fd5a8b3347e7f5557d2ec52ccaf9eb62fa Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 15:22:36 -0600 Subject: [PATCH 09/19] fix(db): retain all ordered joined dependencies --- .../fix-correlated-callback-metadata.md | 2 +- docs/contributing/oracle-coverage.md | 21 ++ .../oracle-reviews/callback-metadata.md | 163 ++++++++++++ packages/db/src/query/compiler/order-by.ts | 16 +- packages/db/src/query/effect.ts | 17 +- packages/db/src/query/live/ARCHITECTURE.md | 7 + .../query/live/collection-config-builder.ts | 24 +- .../src/query/live/collection-subscriber.ts | 7 +- .../src/query/live/ordered-source-loader.ts | 12 +- .../tests/query/ordered-joined-plan-driver.ts | 239 ++++++++++++++++++ ...ordered-source-loader-state-oracle.test.ts | 2 +- .../query/ordered-source-loader-state.test.ts | 2 +- .../ordered-work-oracle.property.test.ts | 121 +++++++++ packages/db/tests/query/ordered-work.test.ts | 2 +- 14 files changed, 600 insertions(+), 35 deletions(-) create mode 100644 packages/db/tests/query/ordered-joined-plan-driver.ts diff --git a/.changeset/fix-correlated-callback-metadata.md b/.changeset/fix-correlated-callback-metadata.md index 386e47dbaf..7ba161e160 100644 --- a/.changeset/fix-correlated-callback-metadata.md +++ b/.changeset/fix-correlated-callback-metadata.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Fix reserved metadata before `fn.select()`, `fn.where()`, and `fn.having()` callbacks in live queries, Effects, and correlated includes. Preserve genuine source metadata, authored inline values, and selected output when compiled queries share a cache. Preserve ordered loading for every window when one source feeds multiple compiled views. +Fix reserved metadata before `fn.select()`, `fn.where()`, and `fn.having()` callbacks in live queries, Effects, and correlated includes. Preserve genuine source metadata, authored inline values, and selected output when compiled queries share a cache. Preserve every ordered window and joined-source publication dependency when one source feeds multiple compiled views or nested queries. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 53b5e8d77d..7e3c6f55ae 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -5964,6 +5964,27 @@ path from a later run. Until then, this entry cannot tell an operator regression from an older defect. A later failure of that campaign must record its seed here before anyone calls it pre-existing. +### Ordered joined dependency composition, October 30 + +October 30 receiving extension: the same ordered-work owner now checks joined +dependency composition across direct, wrapped QueryRef and two-filter nested +ordered plans. Its 128 finite histories cross actual Collection and Effect +receivers, two directions, matched/anti-joined membership, widths one/two, +joined answer order and which of two joined sources vetoes a new head row. +Each history checks settled initial rows, then every recorded publication and +public read at held, partial and settled cuts. All 256 declared held/settled +states reach their minimum once. Unchanged `8464ec977` fails 24 publication +comparisons (12 per receiver); the repair passes all 128 histories. Overwrite, +last-only and first-only dependency mutants fail 24, eight and eight intended +comparisons respectively. The original 424 owner tests and their budgets stay. +This is a controlled adapter witness through real core receivers: one inserted +head row, two joined sources, one QueryRef level and windows of one/two rows. +Ordinary progressive Effect startup callbacks remain outside this new atomicity +claim. No provider/framework, arbitrary nesting, rejection, cleanup or public +window-promise integration claim follows. See the versioned +[joined dependency review](oracle-reviews/callback-metadata.md#ordered-joined-dependency-review-2026-10-30) +for the pre-edit gate, experiments, source binding and loss audit. + ### Ordered signalling-refetch work scope, October 27 Owner: `packages/db/tests/query/ordered-work-oracle.property.test.ts`. The diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 2b77de43c9..911c5db9d2 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -696,3 +696,166 @@ Final SHA-256 binding (the report itself is excluded): | `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts` | `119f75c35a0b5c472d211997a9f677fa0a92e605d99c8bde46aeda8c14a04560` | | `packages/db/src/collection/subscription.ts` | `315a043966bf03ae7671e1e34338692d93eb91f04223a7c079a7585c4a54860b` | | `packages/db/src/query/live/collection-config-builder.ts` | `0c00f76f05ecdea1c842e47e7749ed5dafbbb32f5c9f8a0937faf2776b4f474d` | + + +## Ordered joined dependency review (2026-10-30) + +This entry evaluates CodeRabbit review `5718630084`, run +`bbef8b96-93c0-425c-ac82-76ae3c71abec`, reviewed commit +`86510968691e0080a7484881ed7f54ba58f110af`. Its selected file was +`compiler/basic.test.ts`; the behavioral finding was outside that diff and +survived unchanged at repair parent `8464ec97725a14c9df103e170d9367737c9ef41a`. +The earlier metadata, compiler and replay evidence remains historical evidence +for its named source bindings. This entry supersedes the earlier fingerprints +of the files changed here. The final source binding below identifies this +repair without claiming that the external review covered it. + +**Law and authority.** The architecture's “Ordered requests, continuation, +and recovery” and “Atomic window publication” require ordered continuation +and publication to wait for required joined demand. Retaining two ordered +plans over one source retains both plans' joined dependencies. An outer +QueryRef that has no join cannot remove its child's dependency. A second join +cannot substitute its own source for the child's. This is enforcement of the +existing complete-window law, with no new product decision. + +**Experiment.** The primary `ordered-work-oracle.property.test.ts` owner had +compiler/cache composition coverage and joined receiving coverage separately. +It lacked their intersection: nested ordered plans while joined acquisition +is held. Its new independent judgment filters complete source rows, sorts +and slices using `expectedInstalledWindow`. It never reads the compiler plan +or production publication classifier to compute an expected result. +The supporting `ordered-joined-plan-driver.ts` uses actual live-query +Collections and Effects with controlled source adapters. Adapter request data +selects the marker rows independently through the reference expression judge. +The model's held/partial state abstracts pending acquisition and the preceding +complete public snapshot. The settled state adds the new row and recomputes +membership. Source facts for that row are deliberately unavailable until the +held adapter answers, so the promised early observation is retention, not a +speculative result that needs those missing facts. + +Before production edits, 128 finite histories crossed Collection/Effect, +ascending/descending, matched/anti-joined filters, direct/wrapped/two-filter +nested ordering, widths one/two, source answer order and the source that vetoes +new row 4. Each compares initial settled rows and every recorded change-callback +snapshot or Effect `onBatch` snapshot plus the public read at held, partial +and settled cuts. All 256 declared held/settled states reached once, minimum +one, including both receivers separately. Unchanged parent production failed +24 intended publication comparisons (12 Collection and 12 Effect), with 104 +passes and no type errors. In a wrapped anti-join, row 4 became visible while +its marker acquisition remained held, then disappeared after settlement. +Direct-query controls retained the preceding result correctly. + +The first draft also asserted atomic ordinary Effect startup callbacks. The +architecture explicitly permits progressive startup, so that assertion was +incorrect. It produced two additional failures. Before production edits, the +comparison was corrected to establish the initial settled baseline and begin +its new publication claim afterward. Both earlier logs remain; they do not +supply the authoritative RED result. The final 128-history RED is +`joined-plan-red-final.{json,log}` in the evidence directory below. + +**Repair and calibration.** The compiler stores the union of joined source +IDs when registering ordered plans. Loader continuation, Collection publication +and operation tracking, and Effect publication inspect every dependency. +The new matrix passes all 128 histories. The normal package run passes all +647 tests across the ordered owner and three affected loader/work suites, +including the original 424 owner tests and unchanged fixed/random budgets. +After final type/API-overload and local-name corrections, the same 128-history +matrix passes again. Focused no-emit TypeScript, ESLint (zero errors, five +existing warnings), Prettier, whitespace and changed-file enforcement pass. + +Restoring overwrite semantics fails 24 intended comparisons. A last-only +fallback, equivalent to retaining the new dependency when present, fails eight. +A first-only fallback fails eight. Each narrow control loses the source that +vetoes row 4 in the opposite answer-order cell. All failures are the intended +`ordered joined plan publication mismatch`, not timeouts or setup failures. +All temporary mutants were restored. This rejects both the original defect +and plausible repairs that preserve only one source. + +**Inference and limits.** The finite evidence closes the reported loss of +joined dependencies and its two-source sibling within these legal histories, +actual core receivers and publication cuts. It preserves direct controls, +settled membership, direction and width, and does not add speculative callbacks. +It covers one inserted head row, at most two joined dependencies, one QueryRef +level, widths one/two and controlled successful acquisition. It does not claim +arbitrary nesting, provider/framework timing, new rejection/cleanup laws, +public window-promise integration or atomic ordinary Effect startup. Other +ordered-work limits and existing static metadata/P11/P12 gaps remain recorded +with their prior owners. No known counterexample remains in this new matrix. + +Evidence directory: +`/private/tmp/callback-metadata/ci-monitor/20261030T210918Z`. +It contains the complete raw review/conversation bodies, append-only ledger +references, pre-edit source copies and fingerprints, diagnostic probe, +`joined-plan-red-final.*`, `joined-plan-green.*`, `joined-plan-package-green.*`, +`joined-plan-final-green.*`, three `joined-plan-mutant-*` runs and summary, +`joined-plan-types-final.log`, `joined-plan-eslint-final.log`, formatting and +changed-file enforcement logs. The matrix command is: + +```sh +pnpm_config_verify_deps_before_run=false pnpm --filter @tanstack/db exec \ + vitest run tests/query/ordered-work-oracle.property.test.ts \ + -t 'retains every joined dependency' --coverage.enabled=false --maxWorkers=1 +``` + +### Lossless review ledger and reviewer assessment + +The raw review, repeated prompt, run metadata, mutable risk prose and footnotes +were rechecked against the final diff. Six source-order items remain distinct. +The two-source sibling is calibration and repair work under item 1. + +| ID | Technical verdict and PR action | Durable value and destination | +| --- | --- | --- | +| CR5718630084-1 | **fixed-now**. Dependency loss is real. The review calls it minor; observed transient public membership makes it a correctness defect. Its nullish fallback preserves only one source and fails the wider law. | Nested held-dependency histories in the primary ordered owner, production union and this evidence record. | +| CR5718630084-2 | **duplicate** of 1. The repeated agent prompt proposes the same nullish fallback. | Preserve its exact proposal in the scratch ledger and reject it with the last-only mutant. | +| CR5718630084-3 | **accepted-design**. Optional local CLI review supplies no product requirement and needs no installation. | Existing no-install constraint and explicit oracle calibration remain the authorized verification method. | +| CR5718630084-4 | **accepted-design**. API review commit is 865; one selected test file does not establish fresh review of every production file. The outside-diff finding survives at parent 846. | Keep reviewed commit separate from repair source binding and later CI head. | +| CR5718630084-5 | **duplicate** of 1. Mutable merge-risk prose says to retain the joined dependency. | The receiving oracle now observes the transient public result described by that prose. | +| CR5718630084-6 | **duplicate** of prior review items 5–6. The unchanged 47.37% docstring threshold and optional generated tests do not identify another contract gap. | Preserve repository literate-oracle requirements and review limitations without adding unrelated generated examples. | + +Totals: one fixed-now, two accepted-design, three duplicates, zero +confirmed-open, deferred or design-decision items in this review. The final +loss audit includes prose outside the finding list, optional CLI/tests, +coverage qualification and the mutable summary. No item was removed by a +severity cap. Reviewer recommendation: **hire, with a proposed-fix qualification**. +The review found a real outside-diff composition defect with a precise cause +and useful receiving-test direction. Its priority understated public transient +behavior, and its patch missed two-source composition. The generated policy +footnotes add little new signal. Oracle experiments supply the wider closure. +This assessment does not establish whole-PR readiness or review of the repair. + +### Oracle self-review + +This is a self-review, not an independent external audit. + +| Requirement | Outcome | +| --- | --- | +| ORC-001 | Existing ordered/atomic-publication authority, bounded histories and explicit startup/provider limits above. | +| ORC-002 | Complete source truth, ordinary membership and independent sorting/slicing; no plan or gate helper computes expectations. | +| ORC-003 | Law and limits precede the finite grammar; model, named driver and shared comparison stay distinguishable beside executable prose. | +| ORC-004 | Wrapped anti-join reconstructs the finding. Direct placement removes composition and stays green. Two-filter/blocker/answer-order dimensions distinguish first-only and last-only. Directions, widths and polarities preserve rank/membership neighbors. Bounds are finite; the grammar excludes a second blocker for a plan that has only one joined source. | +| ORC-005 | Real Collection reads/change callbacks and Effect onBatch reconstruction at held, partial and settled cuts; held adapters explain missing source facts without excusing speculative publication. | +| ORC-006 | Overwrite, first-only and last-only controls fail intended public comparisons, with counts 24/eight/eight. | +| ORC-007 | New matrix is bounded enumeration, not a random campaign. Existing owner fixed/random campaigns and budgets remain and pass in the 647-test run. | +| ORC-008, ORC-009 | Pending adapter acquisition maps to held/partial cuts; retained public snapshot and settled source truth are separate model observations. No production continuation machine is copied. | +| ORC-010 | Comparison errors retain observation/history traces in AggregateError and cleanup uses withOracleCleanup. Earlier recorder-assumption failures and all mutant diagnostics remain separately available. | +| ORC-011 | No new shared semantic-fault hypothesis requires a second runtime model. Source-complete membership differs structurally from compiler dependency union and publication scheduling. | +| ORC-012 | Parent, chronology, boundaries, source bindings, loss audit and every applicable outcome are recorded here. | +| ORC-013 | No-join outer plan and two distinct joins distinguish absence preservation from union; both blocker sources and answer orders reject narrow fixes. | +| ORC-014 | Actual core receivers supply the receiving boundary; controlled adapters alone supply acquisition timing. No external provider/framework transfer is claimed. | +| ORC-015 | Every one of 128 histories reaches held and settled once: 256 cells, minimum one, per-receiver identity included in each key. | +| ORC-016 | All new cases use the same independent model, named production driver and refinement comparison; no ordinary expected-answer examples are added to the oracle. | + +Final SHA-256 binding for this repair (this report is excluded): + +| Source | SHA-256 | +| --- | --- | +| `packages/db/src/query/compiler/order-by.ts` | `265c1a8102865e8d0be70bab3b6853e757530800098c8b166545988445b4fab9` | +| `packages/db/src/query/effect.ts` | `a17ed6693a48de79a45b8602bd7803b6a577174d6d796db7038e10c964986220` | +| `packages/db/src/query/live/collection-config-builder.ts` | `fb584c9a49a68faa7775b572ec6d5e0d2ce9f968904d18000c269734c735ab76` | +| `packages/db/src/query/live/collection-subscriber.ts` | `d2e7864f197eaced0323453eda3c70c6967080b956e1ef6df266f1b73234e527` | +| `packages/db/src/query/live/ordered-source-loader.ts` | `3baf426831e184b88656c475be2e6362cd670e6f5fa6e300ea9ec2ed82bf10db` | +| `packages/db/tests/query/ordered-work-oracle.property.test.ts` | `652a1a2378ab882a3e4b0619889020f0581212653507f25051a684b60e8b1d71` | +| `packages/db/tests/query/ordered-joined-plan-driver.ts` | `3a3bb38caef14e649df17ac7cd0eaae312aa483b4801c4d874888b4112c753f9` | +| `packages/db/tests/query/ordered-source-loader-state-oracle.test.ts` | `679449468e358d51469bffe18d159ec62b8557b1c63aeaa3ea4cee2e05628b49` | +| `packages/db/tests/query/ordered-source-loader-state.test.ts` | `23e30afddbfbd5db5979a44f580ac3fc777e5c5ae216584dd7073964080c2925` | +| `packages/db/tests/query/ordered-work.test.ts` | `df0bb6faf5d81f70c1229511dd38180c206d2df0560593550a6220a8e2a2941f` | diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index f0c05baae3..5fa0591b4b 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -67,8 +67,8 @@ export type OrderByOptimizationInfo = { requiresFullSource: boolean /** Whether the ordered source's installed rows are the whole source. */ sourceHoldsAllRows: boolean - /** Source whose lazy demand must settle before a joined-filter page. */ - joinedFilterSourceId?: string + /** Sources whose lazy demand must settle before a joined-filter page. */ + joinedFilterSourceIds?: ReadonlyArray } function warnOrderIndexFallback( @@ -305,7 +305,9 @@ export function processOrderBy( valueExtractorForRawRow: extract, index, orderBy: sourceOrderBy, - joinedFilterSourceId, + joinedFilterSourceIds: joinedFilterSourceId + ? [joinedFilterSourceId] + : undefined, sourceHoldsAllRows: followRefResult.collection.config.syncMode !== `on-demand`, requiresFullSource: @@ -342,6 +344,12 @@ export function processOrderBy( optimizableOrderByCollections[orderBySourceId] = previous ? { ...info, + joinedFilterSourceIds: [ + ...new Set([ + ...(previous.joinedFilterSourceIds ?? []), + ...(info.joinedFilterSourceIds ?? []), + ]), + ], get offset() { return Math.min(previous.offset, info.offset) }, @@ -371,7 +379,7 @@ export function processOrderBy( // loadMoreIfNeeded reads how many rows the window needs. Without an // index, only an on-demand joined filter can leave the window short. const needsCount = - info.joinedFilterSourceId !== undefined && !info.sourceHoldsAllRows + joinedFilterSourceId !== undefined && !info.sourceHoldsAllRows if (index || needsCount) { setSizeCallback = (getSize) => { info.dataNeeded = (boundary) => { diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 4355c7a753..9f1586211d 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -717,8 +717,8 @@ class EffectPipelineRunner { this.unsubscribeCallbacks.add(unsubscribe) if ( - Object.values(this.optimizableOrderByCollections).some( - (info) => info.joinedFilterSourceId === sourceId, + Object.values(this.optimizableOrderByCollections).some((info) => + info.joinedFilterSourceIds?.includes(sourceId), ) ) { this.unsubscribeCallbacks.add( @@ -764,8 +764,9 @@ class EffectPipelineRunner { () => false, undefined, () => - orderByInfo.joinedFilterSourceId !== undefined && - this.hasPendingJoinedWork(orderByInfo.joinedFilterSourceId), + orderByInfo.joinedFilterSourceIds?.some((joinedSourceId) => + this.hasPendingJoinedWork(joinedSourceId), + ) ?? false, ) this.orderedLoaders.set(sourceId, loader) loader.start() @@ -1143,10 +1144,10 @@ class EffectPipelineRunner { } private hasPendingJoinedFilterWork(): boolean { - return Object.values(this.optimizableOrderByCollections).some( - (info) => - info.joinedFilterSourceId !== undefined && - this.hasPendingJoinedWork(info.joinedFilterSourceId), + return Object.values(this.optimizableOrderByCollections).some((info) => + info.joinedFilterSourceIds?.some((sourceId) => + this.hasPendingJoinedWork(sourceId), + ), ) } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 2f9643da05..3f25b55484 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -410,6 +410,13 @@ and normalized compilations separately. Each ordered top-K retains its own window; the source loader's registered plan covers every window that shares that source, including subsequent moves. +Ordered plan aggregation also retains every joined source whose pending demand +gates a retained plan. An outer ordered QueryRef without a join cannot erase +its child's joined dependency. A second ordered join adds its dependency to +the first. Ordered continuation and Collection/Effect publication wait for all +of them. The ordered-work owner checks this composition through actual +Collection and Effect receivers while either joined acquisition is held. + The runtime owner checks select, where and having source inputs through root, wrapped child, direct child and real Effect receivers. Predicate-with-select controls distinguish callback syntax from placement. Collection receivers diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index ee60295643..47687c4c07 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -450,12 +450,12 @@ export class CollectionConfigBuilder< const loadOperation = this.liveQueryCollection?._sync.beginLoadSubsetOperation() for (const info of Object.values(this.optimizableOrderByCollections)) { - if (!info.joinedFilterSourceId) continue - for (const plan of this.lazySourcesCallbacks[info.joinedFilterSourceId] - ?.plans ?? []) { - const participant = this.activeDemands.get(plan.id)?.participant - if (participant?.isPending()) { - this.trackSubsetLoadOperationPromise(participant.promise) + for (const sourceId of info.joinedFilterSourceIds ?? []) { + for (const plan of this.lazySourcesCallbacks[sourceId]?.plans ?? []) { + const participant = this.activeDemands.get(plan.id)?.participant + if (participant?.isPending()) { + this.trackSubsetLoadOperationPromise(participant.promise) + } } } } @@ -1209,10 +1209,10 @@ export class CollectionConfigBuilder< this.orderedLoadFailed || this.hasPendingSourceRecovery() || this.pendingOrderedLoads.size > 0 || - Object.values(this.optimizableOrderByCollections).some( - (info) => - info.joinedFilterSourceId !== undefined && - this.hasPendingJoinedWork(info.joinedFilterSourceId), + Object.values(this.optimizableOrderByCollections).some((info) => + info.joinedFilterSourceIds?.some((sourceId) => + this.hasPendingJoinedWork(sourceId), + ), ) ) { return @@ -1409,8 +1409,8 @@ export class CollectionConfigBuilder< if (status === `ready`) { if ( - Object.values(this.optimizableOrderByCollections).some( - (info) => info.joinedFilterSourceId === sourceId, + Object.values(this.optimizableOrderByCollections).some((info) => + info.joinedFilterSourceIds?.includes(sourceId), ) ) { this.maybeRunGraphFn?.() diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 5fdb3fbe1f..740f9193f7 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -454,10 +454,9 @@ export class CollectionSubscriber< !this.collectionConfigBuilder.hasActiveWindowOperation(), () => this.collectionConfigBuilder.getGraphInputRevision(), () => - orderByInfo.joinedFilterSourceId !== undefined && - this.collectionConfigBuilder.hasPendingJoinedWork( - orderByInfo.joinedFilterSourceId, - ), + orderByInfo.joinedFilterSourceIds?.some((sourceId) => + this.collectionConfigBuilder.hasPendingJoinedWork(sourceId), + ) ?? false, ) this.orderedLoader.start() diff --git a/packages/db/src/query/live/ordered-source-loader.ts b/packages/db/src/query/live/ordered-source-loader.ts index baa705d94b..1b12edf57e 100644 --- a/packages/db/src/query/live/ordered-source-loader.ts +++ b/packages/db/src/query/live/ordered-source-loader.ts @@ -161,7 +161,7 @@ export class OrderedSourceLoader { info.sourceHoldsAllRows && info.index !== undefined && info.orderBy.length === 1 && - info.joinedFilterSourceId === undefined + !info.joinedFilterSourceIds?.length } private get needsOrderingRepair(): boolean { @@ -295,7 +295,10 @@ export class OrderedSourceLoader { this.blockedLoadMore?.windowOperationGeneration continuesOrderedPrefixRepair ||= this.blockedLoadMore?.continuesOrderedPrefixRepair ?? false - if (this.info.joinedFilterSourceId && this.hasPendingJoinedWork()) { + if ( + this.info.joinedFilterSourceIds?.length && + this.hasPendingJoinedWork() + ) { if ( windowOperationGeneration !== undefined || continuesOrderedPrefixRepair @@ -879,7 +882,10 @@ export class OrderedSourceLoader { continuation.continuesOrderedPrefixRepair, ) } else { - if (this.info.joinedFilterSourceId && this.hasPendingJoinedWork()) { + if ( + this.info.joinedFilterSourceIds?.length && + this.hasPendingJoinedWork() + ) { this.stagedContinuation = continuation return } diff --git a/packages/db/tests/query/ordered-joined-plan-driver.ts b/packages/db/tests/query/ordered-joined-plan-driver.ts new file mode 100644 index 0000000000..af92ae19a7 --- /dev/null +++ b/packages/db/tests/query/ordered-joined-plan-driver.ts @@ -0,0 +1,239 @@ +/** Real Collection and Effect receivers for the ordered-work owner. Adapter + * holds control joined acquisition; observations include every publication and + * the current public result at each cut. The owner supplies independent truth. + */ +import { createCollection } from '../../src/collection/index.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { + Query, + createLiveQueryCollection, + eq, + isUndefined, + not, +} from '../../src/query/index.js' +import { createEffect } from '../../src/query/effect.js' +import { evaluateReferenceExpression } from '../reference-expression-oracle.js' +import { flushPromises, withOracleCleanup } from '../utils.js' +import type { LoadSubsetOptions, SyncConfig } from '../../src/types.js' + +export type OrderedJoinedPlanHistory = { + receiver: `collection` | `effect` + direction: `asc` | `desc` + filter: `some` | `none` + placement: `direct` | `wrapped` | `two-filters` + width: 1 | 2 + releaseFirst: `first` | `second` + blocker: `first` | `second` +} +export type OrderedPlanRow = { id: number; rank: number } +export type OrderedJoinedPlanObservation = { + cut: `initial` | `held` | `partial` | `settled` + pending: number + rows: Array + publications: Array> +} +export async function receiveOrderedJoinedPlanHistory( + history: OrderedJoinedPlanHistory, + check: (observation: OrderedJoinedPlanObservation) => void, +) { + type Marker = { id: number; rowId: number } + let rowSync!: Parameters[`sync`]>[0] + let holding = false + const held: Array<{ + source: `first` | `second` + options: LoadSubsetOptions + answer: () => void + }> = [] + const rows = createCollection({ + id: `ordered-plan-rows`, + getKey: (row: OrderedPlanRow) => row.id, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + syncMode: `on-demand`, + sync: { + sync: (operations) => { + rowSync = operations + operations.begin() + for (const id of [1, 2, 3]) + operations.write({ type: `insert`, value: { id, rank: id } }) + operations.commit() + operations.markReady() + return { loadSubset: () => true } + }, + }, + }) + const markerSource = (source: `first` | `second`, marked: Array) => { + const collection = createCollection({ + id: `ordered-plan-${source}`, + getKey: (marker: Marker) => marker.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (operations) => { + operations.markReady() + return { + loadSubset: (options) => { + const answer = () => { + if (options.signal?.aborted) return + operations.begin() + for (const id of marked) { + const value = { id, rowId: id } + if ( + options.where && + evaluateReferenceExpression(options.where, value) !== true + ) + continue + operations.write({ + type: collection.has(id) ? `update` : `insert`, + value, + }) + } + operations.commit() + } + if (holding) + return new Promise((resolve) => + held.push({ + source, + options, + answer: () => { + answer() + resolve() + }, + }), + ) + answer() + return true + }, + } + }, + }, + }) + return collection + } + const firstSource = markerSource( + `first`, + history.blocker === `first` ? [1, 4] : [1], + ) + const secondSource = markerSource( + `second`, + history.blocker === `second` ? [1, 4] : [1], + ) + const inner = new Query() + .from({ row: rows }) + .leftJoin({ first: firstSource }, ({ row, first }) => + eq(row.id, first.rowId), + ) + .where(({ first }) => + history.filter === `none` + ? isUndefined(first.id) + : not(isUndefined(first.id)), + ) + .orderBy(({ row }) => row.rank, history.direction) + .limit(history.placement === `direct` ? history.width : 3) + .select(({ row }) => row) + const query = + history.placement === `direct` + ? inner + : history.placement === `wrapped` + ? new Query() + .from({ nested: inner }) + .orderBy(({ nested }) => nested.rank, history.direction) + .limit(history.width) + .select(({ nested }) => nested) + : new Query() + .from({ nested: inner }) + .leftJoin({ second: secondSource }, ({ nested, second }) => + eq(nested.id, second.rowId), + ) + .where(({ second }) => + history.filter === `none` + ? isUndefined(second.id) + : not(isUndefined(second.id)), + ) + .orderBy(({ nested }) => nested.rank, history.direction) + .limit(history.width) + .select(({ nested }) => nested) + const effectRows = new Map() + let publications: Array> = [] + const readEffect = () => + [...effectRows.values()] + .sort((a, b) => + history.direction === `asc` ? a.rank - b.rank : b.rank - a.rank, + ) + .map(({ id }) => id) + let cleanupReceiver: () => void | Promise = () => {} + let read = readEffect + await withOracleCleanup(async () => { + if (history.receiver === `collection`) { + const live = createLiveQueryCollection(() => query) + read = () => live.toArray.map(({ id }) => id) + const subscription = live.subscribeChanges(() => + publications.push(read()), + ) + cleanupReceiver = async () => { + subscription.unsubscribe() + await live.cleanup() + } + await live.preload() + } else { + const effect = createEffect({ + query: () => query, + onBatch: (events) => { + for (const event of events) { + if (event.type === `exit`) effectRows.delete(event.key) + else effectRows.set(event.key, { ...event.value }) + } + publications.push(readEffect()) + }, + }) + cleanupReceiver = () => effect.dispose() + } + await flushPromises() + const cut = (name: OrderedJoinedPlanObservation[`cut`]) => { + check({ + cut: name, + rows: read(), + pending: held.filter(({ options }) => !options.signal?.aborted).length, + publications, + }) + publications = [] + } + const answer = async (source?: `first` | `second`) => { + for (let turn = 0; turn < 50; turn++) { + const at = held.findIndex( + (load) => source === undefined || load.source === source, + ) + if (at < 0) return + held.splice(at, 1)[0]!.answer() + await flushPromises() + } + throw new Error(`joined plan receiver did not drain`) + } + cut(`initial`) + holding = true + rowSync.begin() + rowSync.write({ + type: `insert`, + value: { id: 4, rank: history.direction === `asc` ? 0 : 4 }, + }) + rowSync.commit() + await flushPromises() + cut(`held`) + await answer(history.releaseFirst) + cut(`partial`) + holding = false + await answer() + await flushPromises() + cut(`settled`) + }, [ + async () => { + holding = false + for (const load of held.splice(0)) load.answer() + await cleanupReceiver() + }, + () => rows.cleanup(), + () => firstSource.cleanup(), + () => secondSource.cleanup(), + ]) +} diff --git a/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts b/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts index df1ceebb1e..4f8f4ef9b1 100644 --- a/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts +++ b/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts @@ -516,7 +516,7 @@ async function checkAuthorityCase( // A synchronous repair answer reads installed rows: an eager source. ...(testCase.syncRepair ? { sourceHoldsAllRows: true } : {}), requiresFullSource: testCase.requiresFullSource, - ...(testCase.joined ? { joinedFilterSourceId: `markers` } : {}), + ...(testCase.joined ? { joinedFilterSourceIds: [`markers`] } : {}), // A window that needs more always needs one row, as behind a joined // filter that drops rows. Otherwise it needs the window's rows that do // not sort at or before the loaded boundary. diff --git a/packages/db/tests/query/ordered-source-loader-state.test.ts b/packages/db/tests/query/ordered-source-loader-state.test.ts index a633d793cb..4e080489a6 100644 --- a/packages/db/tests/query/ordered-source-loader-state.test.ts +++ b/packages/db/tests/query/ordered-source-loader-state.test.ts @@ -93,7 +93,7 @@ describe(`Ordered source request ownership`, () => { let joinedPending = false const loader = new OrderedSourceLoader( createOrderByInfo({ - joinedFilterSourceId: `child`, + joinedFilterSourceIds: [`child`], requiresFullSource: true, }), fakeSubscription(requests, releases), diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 20f2da83d4..6b283bd586 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -21,6 +21,7 @@ import { createOracleReach } from '../oracle-reach.js' import { flushPromises } from '../utils.js' import { withHistoryCleanup } from '../optimistic-history-oracle.js' import { receiveOrderedCompilerCacheHistory } from './ordered-compiler-cache-driver.js' +import { receiveOrderedJoinedPlanHistory } from './ordered-joined-plan-driver.js' import { compareRows, expectDistinctRequests, @@ -8021,3 +8022,123 @@ it.each(compilerCacheWindows)( ) }, ) + +/** Ordered publication waits for every joined dependency that any retained + * source plan requires (ARCHITECTURE.md, atomic window publication). A QueryRef + * adds ordering without removing its child's lazy join. Another ordered join + * adds a dependency rather than replacing it. The model filters complete source + * rows, then sorts and slices. While required adapter loads are held, every + * public read and callback retains the preceding complete result. + * + * This finite history grammar varies receiving host, join polarity, nested + * plan composition, window width, direction and which joined source answers + * first. Direct plans are controls. It uses actual Collection/Effect receivers + * with controlled adapters, not a provider or framework timing claim. + */ +const joinedPlanHistories = ([`collection`, `effect`] as const).flatMap( + (receiver) => + ([`asc`, `desc`] as const).flatMap((direction) => + ([`some`, `none`] as const).flatMap((filter) => + ([`direct`, `wrapped`, `two-filters`] as const).flatMap((placement) => + ([1, 2] as const).flatMap((width) => + ([`first`, `second`] as const).flatMap((releaseFirst) => + (placement === `two-filters` + ? ([`first`, `second`] as const) + : ([`first`] as const) + ).map((blocker) => ({ + receiver, + direction, + filter, + placement, + width, + releaseFirst, + blocker, + })), + ), + ), + ), + ), + ), +) +function expectedJoinedPlanWindow( + history: (typeof joinedPlanHistories)[number], + added: boolean, +) { + const rows = [1, 2, 3].map((id) => ({ id, rank: id })) + if (added) rows.push({ id: 4, rank: history.direction === `asc` ? 0 : 4 }) + const markers = + history.placement === `two-filters` + ? [ + history.blocker === `first` ? [1, 4] : [1], + history.blocker === `second` ? [1, 4] : [1], + ] + : [[1, 4]] + const accepted = rows.filter(({ id }) => + markers.every((marked) => + history.filter === `some` ? marked.includes(id) : !marked.includes(id), + ), + ) + return expectedInstalledWindow(accepted, { + direction: history.direction, + offset: 0, + limit: history.width, + }) +} +declareOracle({ + model: [expectedJoinedPlanWindow], + grammar: [joinedPlanHistories], +}) +const joinedPlanReach = createOracleReach( + `ordered joined plan publication`, + joinedPlanHistories.flatMap((history) => [ + `${JSON.stringify(history)}/held`, + `${JSON.stringify(history)}/settled`, + ]), + { replaying: false }, +) +let joinedPlanEntered = false +afterAll(() => { + if (joinedPlanEntered) joinedPlanReach.assertReached() +}) +it.each(joinedPlanHistories)( + `retains every joined dependency through ordered plans: %j`, + async (history) => { + joinedPlanEntered = true + joinedPlanReach.beginHistory() + const failures: Array = [] + const traces: Array = [] + await receiveOrderedJoinedPlanHistory(history, (observation) => { + const added = observation.cut !== `initial` && observation.pending === 0 + const expected = expectedJoinedPlanWindow(history, added) + try { + expect( + observation.rows, + `${JSON.stringify(history)}/${observation.cut}`, + ).toEqual(expected) + // Ordinary Effect startup may deliver progressive callbacks + // (ARCHITECTURE.md, atomic window publication). This history starts its + // publication claim after readiness; the initial settled rows establish + // the preceding complete result. + if (observation.cut !== `initial`) + for (const publication of observation.publications) + expect( + publication, + `complete ${observation.cut} publication`, + ).toEqual(expected) + } catch (error) { + failures.push(error) + traces.push({ ...observation, expected }) + } + if (observation.cut === `held` && observation.pending > 0) + joinedPlanReach.reach(`${JSON.stringify(history)}/held`) + if (observation.cut === `settled` && observation.pending === 0) + joinedPlanReach.reach(`${JSON.stringify(history)}/settled`) + }) + if (failures.length) + throw new AggregateError( + failures, + `ordered joined plan publication mismatch: ${JSON.stringify(traces)}`, + { cause: failures[0] }, + ) + }, +) diff --git a/packages/db/tests/query/ordered-work.test.ts b/packages/db/tests/query/ordered-work.test.ts index a8353e33cf..a55c61846d 100644 --- a/packages/db/tests/query/ordered-work.test.ts +++ b/packages/db/tests/query/ordered-work.test.ts @@ -207,7 +207,7 @@ describe(`ordered source work oracle`, () => { const builder = getCollectionBuilder(live)! expect( Object.values(builder.optimizableOrderByCollections).some( - (info) => info.joinedFilterSourceId !== undefined, + (info) => (info.joinedFilterSourceIds?.length ?? 0) > 0, ), ).toBe(true) const scheduled = vi.spyOn(builder, `scheduleGraphRun`) From 879b70c9a693901ec67de05e73ed5be90306d1d3 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 15:30:16 -0600 Subject: [PATCH 10/19] test(db): retain sibling replay failure identities --- docs/contributing/oracle-coverage.md | 11 +++ .../oracle-reviews/callback-metadata.md | 64 +++++++++++++++ ...ad-subset-replay-refinement-oracle.test.ts | 77 +++++++++++++++++++ 3 files changed, 152 insertions(+) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 7e3c6f55ae..3232bfca29 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6274,3 +6274,14 @@ controls are separate files. The [chunk 26 record](oracle-reviews/2026-10-28-pr- retains exact source/evidence hashes and all ORC outcomes. Actual production bundle erasure, Typedoc generation, tooling GREEN and restored-old-tooling mutants remain required receiving checks. Generated docs remain CI-owned. + + +### Replay successful-retirement ownership, October 30 + +The load-subset replay refinement owner scopes successful replacement to the +same subscription as the failure. Its 96-history failure/success matrix reaches +four sibling-success coverage histories and calibrates that another subscription's +failure survives the account action. The original account fails that comparison; +all 44 normal owner tests pass after repair, with existing budgets intact. +No production change or new provider/framework claim follows. See the +[versioned retirement review](oracle-reviews/callback-metadata.md#replay-retirement-scope-review-2026-10-30). diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 911c5db9d2..50212577f6 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -859,3 +859,67 @@ Final SHA-256 binding for this repair (this report is excluded): | `packages/db/tests/query/ordered-source-loader-state-oracle.test.ts` | `679449468e358d51469bffe18d159ec62b8557b1c63aeaa3ea4cee2e05628b49` | | `packages/db/tests/query/ordered-source-loader-state.test.ts` | `23e30afddbfbd5db5979a44f580ac3fc777e5c5ae216584dd7073964080c2925` | | `packages/db/tests/query/ordered-work.test.ts` | `df0bb6faf5d81f70c1229511dd38180c206d2df0560593550a6220a8e2a2941f` | + + +## Replay retirement scope review (2026-10-30) + +CodeRabbit review `5718757637`, run `f8b20e61-196f-44d2-86fa-d7648cfa8d08`, +covered `8464ec977`. Three files were selected, two similar documentation files +were skipped, and the replay oracle was processed. The current production +repair `c33e18fd5` was not reviewed. Automatic reviews are now paused. + +**Law.** Each subscription owns its replay verdict (architecture, “Atomic +window publication” and “Replay participants and failure”). A successful +sibling acquisition cannot decide another subscription's replay failure, +even when both request the same source keys. Publication can remain shared. + +**Experiment.** The prior oracle repair scoped error comparison and covering +failures, but missed successful retirement in `settleParticipant`. The primary +owner now varies failure followed by successful coverage over 96 histories: +two directions, four join forms, three widths, root/marker replay and both +settlement orders. It uses the existing actual Collection driver and every +shared public comparison. A calibration comparison also requires every sibling +failure identity to survive that account action. Before repairing the account, +it fails with `replay account scope: successful sibling removed another +subscription's replay failure`, with no type errors. The repair adds the same +subscription premise to successful failure retirement. No production code +changes for this finding. The complete normal owner run passes all 44 tests, +including unchanged fixed/random budgets. Four histories reach successful +sibling coverage, minimum one. The prior two-failure matrix retains 52 own +and six sibling error-order histories. The original retirement behavior is +the rejected hostile control. This establishes account sensitivity, not a +new production defect or universal error-order proof. + +Evidence: `ci-monitor/20261030T212023Z/replay-retirement-{red,package-green}.*` +under `/private/tmp/callback-metadata`, original oracle copy, source ledger, +ESLint and changed-file enforcement logs. Existing controlled-adapter, peer, +provider and framework limits remain. The publication, convergence and +request-error comparisons remain, without a new waiver. + +The six-item raw-body loss audit yields one **fixed-now** (successful retirement +scope), two **accepted-design** (coverage qualification and paused review), +and three **duplicates** (repeated fix prompt, mutable risk prose, docstring +policy and optional generated docs/tests). The docstring percentage changes to +45%, but names no separate missing contract explanation. Optional review +commands supply no coverage and are not executed. No item is deferred or open. +Reviewer recommendation: **hire**. The finding correctly identifies a remaining +oracle-integrity gap and proposes the missing ownership premise. Its scope is +test correctness. It does not establish review of the later production repair. + +ORC self-review, not independent audit: 001–003 use the established ownership +law, input identities and literate shared comparison. 004 reconstructs the +failure/success neighbor of the prior failure/failure grammar, with finite +bounds and no peer claim. 005 exercises actual Collections at the existing +cuts and calibrates the oracle account separately. 006 rejects the original +retirement action at its intended assertion. 007 preserves all campaigns and +budgets. 008–009 retain per-subscription failure identities within the shared +publication abstraction. 010 retains the original trace and existing cleanup. +011 names no new shared semantic-fault hypothesis requiring a second runtime +model. 012 records chronology, source and limits here. 013 crosses failure with +success and own with sibling coverage. 014 claims controlled adapters through +real core receivers only. 015 reaches four successful sibling histories against +minimum one. 016 supplies histories to the shared driver and judgments, with +no separate expected Error examples. + +Final SHA-256 of `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts`: +`e03f3cee17413016f59c299eb25896fd9098ea3cde88fffbe35f4de58fba366f`. This supersedes that owner's earlier binding only. diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts index 248bae88c3..805eafe3af 100644 --- a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts @@ -319,6 +319,7 @@ type OrderedReach = { deferredMoveRejections: number ownReplayFailureComparisons: number siblingReplayFailureComparisons: number + siblingSuccessfulCoverage: number replayLoads: number abortRejections: number preloadChecks: number @@ -545,6 +546,7 @@ async function checkOrderedReplay( barrier.own.add(load) return true } + let replayScopeViolation: string | undefined const settleParticipant = ( source: BarrierSource, load: Load, @@ -574,15 +576,32 @@ async function checkOrderedReplay( // failure no longer fails the replay (ARCHITECTURE.md law 17: a failure // counts for a key no later held load loaded). if (ok && barrier?.pending.has(load) && !options.signal?.aborted) { + // Calibration of the replay account: a successful sibling can supply the + // same source keys without deciding another subscription's verdict. + // Preserve those failure identities independently of the mutation below. + const siblingFailures = [...barrier.failures.keys()].filter( + (failed) => failed.subscription !== load.subscription, + ) for (const failed of barrier.failures.keys()) { + if ( + failed.subscription !== load.subscription && + loads.indexOf(failed) < loads.indexOf(load) && + failed.ordered === load.ordered && + failed.keys !== undefined && + [...failed.keys].every((key) => load.keys?.has(key)) + ) + reach.siblingSuccessfulCoverage++ if ( loads.indexOf(failed) < loads.indexOf(load) && + failed.subscription === load.subscription && failed.ordered === load.ordered && failed.keys !== undefined && [...failed.keys].every((key) => load.keys?.has(key)) ) barrier.failures.delete(failed) } + if (siblingFailures.some((failed) => !barrier.failures.has(failed))) + replayScopeViolation ??= `successful sibling removed another subscription's replay failure` } barrier?.pending.delete(load) // A replay completes when its last participant settles, so demand that @@ -897,6 +916,7 @@ async function checkOrderedReplay( deferredMoveRejections: 0, ownReplayFailureComparisons: 0, siblingReplayFailureComparisons: 0, + siblingSuccessfulCoverage: 0, replayLoads: 0, abortRejections: 0, preloadChecks: 0, @@ -1731,6 +1751,8 @@ async function checkOrderedReplay( if (moveOrderViolation) { throw new Error(`newest move wins: ${moveOrderViolation}`) } + if (replayScopeViolation) + throw new Error(`replay account scope: ${replayScopeViolation}`) if (moveErrorViolation) { throw new Error(`replay error identity: ${moveErrorViolation}`) } @@ -2525,6 +2547,59 @@ describe(`Ordered windows across truncate replays`, () => { declared.assertReached() }, 120_000) + // Failure followed by successful coverage is the neighboring regime of + // the two-failure grammar above. Real adapter subscriptions supply identity; + // the shared account comparison requires a sibling's failure to survive. + it(`retains sibling replay failures through successful coverage`, async () => { + const declared = createOracleReach( + `replay.subscription-success-scope`, + [`siblingSuccessfulCoverage`] as const, + { minimum: 1 }, + ) + for (const direction of [`asc`, `desc`] as const) { + for (const joined of [false, `some`, `none`, `inner`] as const) { + for (const limit of [1, 2, 3] as const) { + for (const markers of [false, true]) { + for (const newest of [false, true]) { + declared.beginHistory() + const observed = await checkOrderedReplay( + [ + { type: `hold` }, + markers + ? { type: `truncateMarkers` } + : { type: `truncate`, armAfterStart: false }, + { type: `window`, offset: 0, limit }, + { type: `settle`, ok: false, newest, keepHolding: true }, + { + type: `settle`, + ok: true, + newest: !newest, + keepHolding: false, + }, + ], + { + ...ordinaryOrderedScenario, + direction, + joined, + limit, + markerMode: `on-demand`, + includes: true, + }, + ) + if (observed.siblingSuccessfulCoverage > 0) + declared.reach(`siblingSuccessfulCoverage`) + } + } + } + } + } + console.info( + `replay.subscription-success-scope`, + Object.fromEntries(declared.counts()), + ) + declared.assertReached() + }, 120_000) + it.each(orderedReplayCampaigns)( `keeps the ordered window under load and release faults (%s campaign)`, async (campaign) => { @@ -2553,6 +2628,7 @@ describe(`Ordered windows across truncate replays`, () => { deferredMoveRejections: 0, ownReplayFailureComparisons: 0, siblingReplayFailureComparisons: 0, + siblingSuccessfulCoverage: 0, replayLoads: 0, abortRejections: 0, preloadChecks: 0, @@ -2593,6 +2669,7 @@ describe(`Ordered windows across truncate replays`, () => { `deferredMoveRejections`, `ownReplayFailureComparisons`, `siblingReplayFailureComparisons`, + `siblingSuccessfulCoverage`, `successWhileSubscriptionOwes`, ]) // Counts are histories, rather than repeated events in one trace. From 1e62c7a9cd4cf8d59dae083d72b0a413aa46f84b Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 15:59:34 -0600 Subject: [PATCH 11/19] refactor(db): reduce compiler shipped size --- .../oracle-reviews/callback-metadata.md | 48 ++ packages/db/src/query/compiler/index.ts | 778 +++++------------- packages/db/src/query/compiler/joins.ts | 701 +++++++--------- packages/db/src/query/compiler/order-by.ts | 126 +-- .../db/src/query/compiler/route-metadata.ts | 161 ++-- 5 files changed, 662 insertions(+), 1152 deletions(-) diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 50212577f6..9ae2c48ca4 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -923,3 +923,51 @@ no separate expected Error examples. Final SHA-256 of `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts`: `e03f3cee17413016f59c299eb25896fd9098ea3cde88fffbe35f4de58fba366f`. This supersedes that owner's earlier binding only. + + +## Shipped-size refactor (2026-10-30) + +The size request preserves the accepted metadata, cache, ordering and publication +laws. The primary owners, history grammars, comparisons and run budgets do not +change. This is a behavior-preserving refactor, with no new pre-edit RED claim. +Shared FROM/JOIN source compilation and input wrapping replace duplicate paths. +Container copying, include replacement, projection and ordering each use fewer +implementations. Authored output and normalized callback views remain distinct. +Every ordered window and joined dependency remains represented. + +`pnpm size:compare --packages db --files` measures production bundles against +merge-base `bfd1da1a7e752d66fd667adfd3cdf54f60732b22`. The repository has no +`size:check` script. Targeted builds use existing dependencies without an install. + +| Measurement | Base gzip | Refactor gzip | Gzip delta | Brotli delta | +| --- | ---: | ---: | ---: | ---: | +| Full `@tanstack/db` package | 109,702 B | 109,673 B | −29 B | −13 B | +| Typical app | 96,777 B | 96,743 B | −34 B | −26 B | + +These totals do not establish that every feature scenario shrinks. LocalStorage +adds 17 B gzip and 5 B Brotli. Some other scenarios shrink with gzip but grow +with Brotli. The complete scenario table remains in `size-union-key.log`. + +The frozen source passes 6,322 normal-package tests across 11 suites, including +both primary owners and the affected compiler, container and include neighbors. +Focused no-emit TypeScript, ESLint (zero errors and warnings), Prettier, +changed-file enforcement and whitespace checks pass. Seven source-bound hostile +controls fail their intended comparisons: lost HAVING view 551, over-normalized +predicates 696, raw-only cache 26, ordered-window overwrite 48, joined-dependency +overwrite 24, joined last-only eight and joined first-only eight. These controls +calibrate the existing laws after refactoring. They do not extend their limits. +Union key encoding also matches the original for 24 representative inputs. + +Evidence directory: +`/private/tmp/callback-metadata/ci-monitor/20261030T212503Z/size-reduction`. +The final bindings are `final-green.*`, `final-eslint2.log`, `final-types2.log`, +`final-format.log`, `final-lint-changed.log`, `size-mutant-summary.json` and +`final-fingerprints.json`. Earlier attempts remain separate. The following +bindings supersede earlier bindings for these four refactored files only. + +| Source | SHA-256 | +| --- | --- | +| `packages/db/src/query/compiler/index.ts` | `c6074cd82e6755b3d55c9307018fe368cd3e591a719eeb69ab6d7cb2109e75bf` | +| `packages/db/src/query/compiler/joins.ts` | `a5618bb1da3bff18b91fc4d9bf155447712e0bb1b68cd0f66a1cda0662d80450` | +| `packages/db/src/query/compiler/order-by.ts` | `a104d4f4829e9b4ea801fcc60914772c1d97a47d7d46dbfa5e24f49724b12267` | +| `packages/db/src/query/compiler/route-metadata.ts` | `bc6313f715e84949d0a6f8f5b48511bb3bc05ba5fd13a0b5e8a1033f3ee8e4df` | diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index a7c1964593..9b16d78850 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -19,7 +19,6 @@ import { getParentContextValue, } from '../equality-value-identity.js' import { - CollectionInputNotFoundError, DistinctRequiresSelectError, FnSelectWithGroupByError, GroupingOperatorRequiredError, @@ -56,7 +55,6 @@ import { getWhereExpression, isBasicOrAggregateExpression, isExpressionLike, - requireCollectionSource, } from '../ir.js' import { ensureIndexForField } from '../../indexes/auto-index.js' import { createSourceRecord } from '../../utils/source-record.js' @@ -68,12 +66,11 @@ import { isCaseWhenConditionTrue, toBooleanPredicate, } from './evaluators.js' -import { processJoins, registerLazyDemandPlan } from './joins.js' +import { processJoins, processSource, registerLazyDemandPlan } from './joins.js' import { findAggregate } from './find-aggregate.js' import { getLazyLoadTargets } from './lazy-targets.js' import { processOrderBy } from './order-by.js' import { crossJoinParentRoutes } from './parent-routes.js' -import { queriesMatchForCaching } from './query-equivalence.js' import { CALLBACK_INPUT, CALLBACK_SELECTED, @@ -85,10 +82,11 @@ import { getNamespacedRouteMetadata, getRouteMetadata, getRoutedScalarMetadata, + isPublicContainer, stripInternalCallbackMetadata, stripInternalRouteMetadata, - stripRouteMetadata, transformPublicContainers, + wrapInputRow, } from './route-metadata.js' import { processSelect } from './select.js' import type { ValueIdentity } from '../equality-value-identity.js' @@ -144,12 +142,9 @@ function getUnsupportedFnSelectResultDescription( } seen.add(value) - const keys = [ - ...Object.keys(value), - ...Object.getOwnPropertySymbols(value).filter((key) => - Object.prototype.propertyIsEnumerable.call(value, key), - ), - ] + const keys = Reflect.ownKeys(value).filter((key) => + Object.prototype.propertyIsEnumerable.call(value, key), + ) for (const key of keys) { const entry = (value as Record)[key] const unsupported = getUnsupportedFnSelectResultDescription(entry, seen) @@ -282,14 +277,6 @@ function parameterizeByParentRoutes( ) as NamespacedAndKeyedStream } -function getRowCorrelationKey(row: NamespacedRow, mainSource: string): unknown { - return getNamespacedRouteMetadata(row, mainSource)?.correlationKey -} - -function getRowParentContext(row: NamespacedRow, mainSource: string): unknown { - return getNamespacedRouteMetadata(row, mainSource)?.parentContext ?? null -} - function correlationValuesEqual(left: unknown, right: unknown): boolean { if (left == null || right == null) return false const normalizedLeft = normalizeValue(left) @@ -625,17 +612,30 @@ function compileQueryInternal( mainCollectionId, mainSource, allInputs, - cache, queryMapping, collections, - subscriptions, callbacks, lazySources, - optimizableOrderByCollections, - ignoreWindow, rawQuery, query.from, - (...args) => compileQueryInternal(args, true, sanitizeSourceMetadata), + (subquery, parentRoutes) => + compileQueryInternal( + [ + subquery, + allInputs, + collections, + subscriptions, + callbacks, + lazySources, + optimizableOrderByCollections, + ignoreWindow, + cache, + queryMapping, + parentRoutes, + ], + true, + sanitizeSourceMetadata, + ), aliasToCollectionId, aliasRemapping, sourceWhereClauses, @@ -661,7 +661,7 @@ function compileQueryInternal( filter(([, row]) => correlationValuesEqual( compiledChildCorrelation(row), - getRowCorrelationKey(row, mainSource), + getNamespacedRouteMetadata(row, mainSource)?.correlationKey, ), ), ) as NamespacedAndKeyedStream @@ -972,7 +972,7 @@ function compileQueryInternal( // Replace includes entry in select with a null placeholder query = { ...query, - select: replaceIncludesInSelectValue(query.select!, path).value, + select: replaceIncludesInSelectValue(query.select!, path), } } } @@ -1056,61 +1056,40 @@ function compileQueryInternal( // Process the SELECT clause early - always create $selected // This eliminates duplication and allows for DISTINCT implementation if (query.fnSelect) { - const fnSelect = (row: NamespacedRow) => { - const selected = query.fnSelect!(row) - validateFnSelectResult(selected) - return selected - } const sourceAliases = [...sourceBindings.keys()] // Handle functional select - apply the function to transform the row - const projectRow = (namespacedRow: NamespacedRow) => { - const callbackRow = stripInternalCallbackMetadata( - getCallbackInput(namespacedRow), - ) as NamespacedRow - const selectResults = fnSelect(callbackRow) - let selected = selectResults - if ( - selectResults && - typeof selectResults === `object` && - (Array.isArray(selectResults) || isPlainObject(selectResults)) - ) { - selected = Array.isArray(selectResults) - ? [...selectResults] - : { ...selectResults } - if ( - sanitizeSelectedMetadata && - !sourceAliases.some((alias) => selectResults === callbackRow[alias]) - ) { - // New functional objects are projections. Equal field values cannot - // prove which source (if any) authored them. - for (const prop of VIRTUAL_PROP_NAMES) delete selected[prop] - } - const routing = (namespacedRow as any)[INCLUDES_ROUTING] - if (routing) { - selected[INCLUDES_ROUTING] = routing - } - } - let callbackSelected = selected - if ( - selected && - typeof selected === `object` && - (Array.isArray(selected) || isPlainObject(selected)) && - !sourceAliases.some((alias) => selectResults === callbackRow[alias]) - ) { - for (const prop of VIRTUAL_PROP_NAMES) { - if (!(prop in selected)) continue - if (callbackSelected === selected) - callbackSelected = copySelectedRow(selected) - delete callbackSelected[prop] + pipeline = pipeline.pipe( + map(([key, namespacedRow]) => { + const callbackRow = stripInternalCallbackMetadata( + getCallbackInput(namespacedRow), + ) as NamespacedRow + const selectResults = query.fnSelect!(callbackRow) + validateFnSelectResult(selectResults) + let selected = selectResults + if (isPublicContainer(selectResults)) { + selected = copySelectedRow(selectResults) + const routing = (namespacedRow as any)[INCLUDES_ROUTING] + if (routing) selected[INCLUDES_ROUTING] = routing } - } - return { - ...namespacedRow, - $selected: selected, - [CALLBACK_SELECTED]: callbackSelected, - } - } - pipeline = pipeline.pipe(map(([key, row]) => [key, projectRow(row)])) + // A returned source row keeps its genuine metadata. A new functional + // projection has no source owner; use the same copy-on-write metadata + // normalization as structured selections for its callback view. + const callbackSelected = sourceAliases.some( + (alias) => selectResults === callbackRow[alias], + ) + ? selected + : attachVirtualPropsToSelected(selected, {}, true, []) + if (sanitizeSelectedMetadata) selected = callbackSelected + return [ + key, + { + ...namespacedRow, + $selected: selected, + [CALLBACK_SELECTED]: callbackSelected, + }, + ] + }), + ) } else if (query.select) { // An aggregate query computes its select values from each group, so it // does not project rows first. @@ -1144,9 +1123,7 @@ function compileQueryInternal( if (includesRoutingFns.length > 0) { pipeline = pipeline.pipe( map(([key, namespacedRow]: any) => { - const selected = Array.isArray(namespacedRow.$selected) - ? [...namespacedRow.$selected] - : { ...namespacedRow.$selected } + const selected = copySelectedRow(namespacedRow.$selected) selected[INCLUDES_ROUTING] = getRowIncludesRouting(namespacedRow) return [key, { ...namespacedRow, $selected: selected }] }), @@ -1254,14 +1231,15 @@ function compileQueryInternal( : [mainSource] const finalizeRow = ( key: unknown, - row: Record, + row: Record, orderByIndex: string | undefined, ) => { + const selectedValue = unwrapValue(row.$selected) const authoredResults = query.fnSelect && sanitizeSelectedMetadata - ? unwrapValue(row.$selected) + ? selectedValue : attachVirtualPropsToSelected( - unwrapValue(row.$selected), + selectedValue, row, sanitizeSelectedMetadata, metadataSource, @@ -1275,7 +1253,7 @@ function compileQueryInternal( ? (row as { [CALLBACK_SELECTED]?: unknown })[CALLBACK_SELECTED] : attachVirtualPropsToSelected( transformPublicContainers( - unwrapValue(row.$selected), + selectedValue, (value) => sourceViews.find(([alias]) => value === row[alias])?.[1] ?? value, NO_OMITTED_KEYS, @@ -1284,29 +1262,23 @@ function compileQueryInternal( true, metadataSource, ) + const route = parentKeyStream + ? getNamespacedRouteMetadata(row, mainSource) + : undefined // Carry both views with the occurrence. A downstream source adapter selects // the callback view without rewriting the authored inline result. - if (parentKeyStream) { - return [ - key, - [ - finalResults, - orderByIndex, - getRowCorrelationKey(row, mainSource), - getRowParentContext(row, mainSource), - getIncludesPublicKey(row, mainSource, key), - stripInternalRouteMetadata(callbackSelected), - ], - ] as any - } return [ key, [ finalResults, orderByIndex, - undefined, - undefined, - undefined, + route?.correlationKey, + parentKeyStream ? (route?.parentContext ?? null) : undefined, + parentKeyStream + ? (row[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? + row[INCLUDES_PUBLIC_KEY] ?? + key) + : undefined, stripInternalRouteMetadata(callbackSelected), ], ] as unknown as [unknown, [any, string | undefined]] @@ -1332,14 +1304,9 @@ function compileQueryInternal( parentKeyStream && (resultLimit !== undefined || query.offset !== undefined) ? (_key: unknown, row: unknown) => { - const correlationKey = getRowCorrelationKey( - row as NamespacedRow, - mainSource, - ) - const parentContext = getRowParentContext( - row as NamespacedRow, - mainSource, - ) + const route = getNamespacedRouteMetadata(row, mainSource) + const correlationKey = route?.correlationKey + const parentContext = route?.parentContext if (parentContext != null) { return serializeValue([ valueIdentity.equality(correlationKey), @@ -1408,10 +1375,9 @@ function keyWhereClausesBySource( query: QueryIR, clauses: Map>, ): Map> { - const localSources = [ - ...getFromSources(query.from), - ...(query.join?.map(({ from }) => from) ?? []), - ].filter((source): source is CollectionRef => source.type === `collectionRef`) + const localSources = getAllSources(query).filter( + (source): source is CollectionRef => source.type === `collectionRef`, + ) const localSourceByAlias = new Map( localSources.map((source) => [source.alias, source]), ) @@ -1445,17 +1411,20 @@ function canonicalizeSelectedRows( const compiledOrder = (query.orderBy ?? []).map(({ expression }) => compileExpression(expression), ) - const signature = (row: any) => ({ - value: row.$selected, - routing: row.$selected?.[INCLUDES_ROUTING], - outerCorrelation: isIncludedRelation - ? getRowCorrelationKey(row, mainSource) - : undefined, - parentContext: isIncludedRelation - ? getRowParentContext(row, mainSource) - : undefined, - order: compiledOrder.map((evaluate) => evaluate(row)), - }) + const signature = (row: any) => { + const route = isIncludedRelation + ? getNamespacedRouteMetadata(row, mainSource) + : undefined + return { + value: row.$selected, + routing: row.$selected?.[INCLUDES_ROUTING], + outerCorrelation: route?.correlationKey, + parentContext: isIncludedRelation + ? (route?.parentContext ?? null) + : undefined, + order: compiledOrder.map((evaluate) => evaluate(row)), + } + } return pipeline.pipe( reduce((values: Array<[any, number]>) => { @@ -1689,25 +1658,51 @@ function processFromClause( ) } + const compileSource = (source: CollectionSourceRef | QueryRef) => { + const result = processSource( + source, + allInputs, + queryMapping, + (subquery, parentRoutes) => + compileQueryInternal( + [ + subquery, + allInputs, + collections, + subscriptions, + callbacks, + lazySources, + optimizableOrderByCollections, + setWindowFn, + cache, + queryMapping, + parentRoutes, + ], + true, + sanitizeSelectedMetadata, + ), + aliasToCollectionId, + aliasRemapping, + sourceWhereClauses, + valueIdentity, + `from`, + parentKeyStream, + ) + return { + ...result, + sourceIncludes: + result.includes?.map((include) => ({ + sourceAlias: source.alias, + include, + })) ?? [], + isParentRouted: + source.type === `queryRef` && parentKeyStream !== undefined, + } + } + if (from.type !== `unionFrom`) { const { alias, input, collectionId, sourceIncludes, isParentRouted } = - processFrom( - from, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - aliasToCollectionId, - aliasRemapping, - sourceWhereClauses, - parentKeyStream, - sanitizeSelectedMetadata, - ) + compileSource(from) return { alias, @@ -1738,23 +1733,7 @@ function processFromClause( collectionId, sourceIncludes: childSourceIncludes, isParentRouted, - } = processFrom( - source, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - aliasToCollectionId, - aliasRemapping, - sourceWhereClauses, - parentKeyStream, - sanitizeSelectedMetadata, - ) + } = compileSource(source) if (!mainAlias) { mainAlias = alias @@ -1944,230 +1923,20 @@ function wrapInputWithAlias( alias: string, ): NamespacedAndKeyedStream { return input.pipe( - map(([key, row, callbackValue]: [...InputRow, unknown?]) => { - const finish = (namespaced: NamespacedRow): [unknown, NamespacedRow] => { - if (callbackValue !== undefined && callbackValue !== row) { - Object.assign(namespaced, { - [CALLBACK_INPUT]: { [alias]: callbackValue }, - }) - } - return [key, namespaced] - } - const inputRow: unknown = row - const scalar = getRoutedScalarMetadata(inputRow) - if (scalar) { - const nsRow = attachRouteMetadata( - { - ...getParentContextValue(scalar.parentContext), - [alias]: scalar.value, - [INCLUDES_PUBLIC_KEY]: scalar.publicKey, - }, - scalar.correlationKey, - scalar.parentContext, - ) as unknown as NamespacedRow - return finish(nsRow) - } - - if (inputRow == null || typeof inputRow !== `object`) { - return finish({ [alias]: inputRow } as NamespacedRow) - } - - // Initialize the record with a nested structure. Route metadata remains - // outside the user namespace while projected parent aliases stay visible. - const route = getRouteMetadata(inputRow) - const cleanRow = route - ? stripRouteMetadata(inputRow as Record) - : inputRow - const nsRow: Record = { - ...getParentContextValue(route?.parentContext), - [alias]: cleanRow, - } - if (route) { - attachRouteMetadata(nsRow, route.correlationKey, route.parentContext) - } - return finish(nsRow) - }), + map(([key, row, callbackValue]: [...InputRow, unknown?]) => [ + key, + wrapInputRow(alias, row, callbackValue), + ]), ) } function encodeKeyForUnionBranch(key: unknown): string { - if (typeof key === `string`) { - return `string:${key}` - } - if (typeof key === `number`) { - return `number:${String(key)}` - } - if (typeof key === `bigint`) { - return `bigint:${String(key)}` - } - return `${typeof key}:${JSON.stringify(key)}` -} - -function processFrom( - from: CollectionSourceRef | QueryRef, - allInputs: Record, - collections: Record, - subscriptions: Record, - callbacks: Record, - lazySources: Set, - optimizableOrderByCollections: Record, - setWindowFn: (windowFn: (options: WindowOptions) => void) => void, - cache: QueryCache, - queryMapping: QueryMapping, - aliasToCollectionId: Record, - aliasRemapping: Record, - sourceWhereClauses: Map>, - parentKeyStream?: KeyedStream, - sanitizeSelectedMetadata = false, -): { - alias: string - input: KeyedStream - collectionId: string - sourceIncludes: Array - isParentRouted: boolean -} { - switch (from.type) { - case `descriptorRef`: - return requireCollectionSource(from) - case `collectionRef`: { - const input = allInputs[from.sourceId] - if (!input) { - throw new CollectionInputNotFoundError( - from.alias, - from.collection.id, - Object.keys(allInputs), - ) - } - aliasToCollectionId[from.alias] = from.collection.id - return { - alias: from.alias, - input, - collectionId: from.collection.id, - sourceIncludes: [], - isParentRouted: false, - } - } - case `queryRef`: { - // Preserve the user-defined query as the cache key when optimization - // only copied it. If the optimizer changed the query, compile that IR; - // substituting its origin would discard pushed predicates. - const originalQuery = queryMapping.get(from.query) - const queryToCompile = - originalQuery && queriesMatchForCaching(from.query, originalQuery) - ? originalQuery - : from.query - const subQueryResult = compileQueryInternal( - [ - queryToCompile, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - parentKeyStream, - undefined, - ], - true, - sanitizeSelectedMetadata, - ) - - // Pull up alias mappings from subquery to parent scope. - // This includes both the innermost alias-to-collection mappings AND - // any existing remappings from nested subquery levels. - Object.assign(aliasToCollectionId, subQueryResult.aliasToCollectionId) - Object.assign(aliasRemapping, subQueryResult.aliasRemapping) - - // Pull up source WHERE clauses from subquery to parent scope. - // This enables loadSubset to receive the correct where clauses for subquery collections. - // - // IMPORTANT: Skip pull-up for optimizer-created subqueries. These are detected when: - // 1. The outer alias (from.alias) matches the inner alias (from.query.from.alias) - // 2. The subquery was found in queryMapping (it's a user-defined subquery, not optimizer-created) - // - // For optimizer-created subqueries, the parent already has the sourceWhereClauses - // extracted from the original raw query, so pulling up would be redundant. - // More importantly, pulling up for optimizer-created subqueries can cause issues - // when the optimizer has restructured the query. - const isUserDefinedSubquery = queryMapping.has(from.query) - const subqueryFromAlias = getFirstFromAlias(from.query.from) - const isOptimizerCreated = - !isUserDefinedSubquery && from.alias === subqueryFromAlias - - if (!isOptimizerCreated) { - for (const [alias, whereClause] of subQueryResult.sourceWhereClauses) { - sourceWhereClauses.set(alias, whereClause) - } - } - - // Create a FLATTENED remapping from outer alias to innermost alias. - // For nested subqueries, this ensures one-hop lookups (not recursive chains). - // - // Example with 3-level nesting: - // Inner: .from({ user: usersCollection }) - // Middle: .from({ activeUser: innerSubquery }) → creates: activeUser → user - // Outer: .from({ author: middleSubquery }) → creates: author → user (not author → activeUser) - // - // The key insight: We search through the PULLED-UP aliasToCollectionId (which contains - // the innermost 'user' alias), so we always map directly to the deepest level. - // This means aliasRemapping[alias] is always a single lookup, never recursive. - // Needed for subscription resolution during lazy loading. - const innerAlias = Object.keys(subQueryResult.aliasToCollectionId).find( - (alias) => - subQueryResult.aliasToCollectionId[alias] === - subQueryResult.collectionId, - ) - if (innerAlias && innerAlias !== from.alias) { - aliasRemapping[from.alias] = innerAlias - } - - // Extract the pipeline from the compilation result - const subQueryInput = subQueryResult.pipeline - - // Subqueries may return [key, [value, orderByIndex]] (with ORDER BY) or [key, value] (without ORDER BY) - // We need to extract just the value for use in parent queries - const extractedInput = subQueryInput.pipe( - map((data: any) => { - const [ - key, - [ - value, - _orderByIndex, - correlationKey, - parentContext, - publicKey, - callbackValue, - ], - ] = data - // Unwrap Value expressions that might have leaked through as the entire row - const unwrapped = attachRouteMetadataToResult( - unwrapValue(value), - correlationKey, - parentContext, - publicKey, - ) - return [key, unwrapped, callbackValue] as unknown as InputRow - }), - ) - - return { - alias: from.alias, - input: extractedInput, - collectionId: subQueryResult.collectionId, - sourceIncludes: - subQueryResult.includes?.map((include) => ({ - sourceAlias: from.alias, - include, - })) ?? [], - isParentRouted: parentKeyStream !== undefined, - } - } - default: - throw new UnsupportedFromTypeError((from as any).type) - } + const type = typeof key + const value = + type === `string` || type === `number` || type === `bigint` + ? String(key) + : JSON.stringify(key) + return `${type}:${value}` } // Only compiler-created Value instances are wrappers; user rows may have @@ -2191,7 +1960,9 @@ function getWholeRowSourceAliases(select: unknown): Array { ) } -function copySelectedRow(selected: Record) { +function copySelectedRow( + selected: Record, +): Record { return Array.isArray(selected) ? [...selected] : { ...selected } } @@ -2201,13 +1972,7 @@ function attachVirtualPropsToSelected( isCollectionRoot: boolean, metadataSources: ReadonlyArray | undefined, ): any { - if ( - !selected || - typeof selected !== `object` || - (!Array.isArray(selected) && !isPlainObject(selected)) - ) { - return selected - } + if (!isPublicContainer(selected)) return selected const selectedRecord = selected as Record // Union output is already a branch row, rather than an aliased source. @@ -2224,43 +1989,22 @@ function attachVirtualPropsToSelected( : undefined let result = selectedRecord for (const prop of VIRTUAL_PROP_NAMES) { - if (isCollectionRoot && prop in selectedRecord) { - // An explicit virtual name in a root projection cannot replace source - // metadata. Ordinary projections acquire their own Collection metadata. - const owner = - sourceRecord && prop in sourceRecord ? sourceRecord : undefined - if (owner) { - const value = owner[prop] - if (selectedRecord[prop] !== value) { - if (result === selectedRecord) - result = copySelectedRow(selectedRecord) - result[prop] = value - } - } else { - if (result === selectedRecord) result = copySelectedRow(selectedRecord) - delete result[prop] - } - } else if (selectedRecord[prop] == null && prop in row) { - if (result === selectedRecord) result = copySelectedRow(selectedRecord) - result[prop] = row[prop] - } + // A root projection cannot replace source metadata. Other projections + // only inherit a missing virtual property from their namespace. + const reset = isCollectionRoot && prop in selectedRecord + if (!reset && (selectedRecord[prop] != null || !(prop in row))) continue + const owner = reset ? sourceRecord : row + const present = owner !== undefined && prop in owner + const value = present ? owner[prop] : undefined + if (reset && present && selectedRecord[prop] === value) continue + if (result === selectedRecord) result = copySelectedRow(selectedRecord) + if (present) result[prop] = value + else delete result[prop] } return result } -function getIncludesPublicKey( - row: Record, - mainSource: string, - fallback: unknown, -): unknown { - return ( - row[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? - (row as any)[INCLUDES_PUBLIC_KEY] ?? - fallback - ) -} - /** * Recursively records the user-defined origin of optimized subqueries. * Compilation still consumes the optimized query; the mapping is lineage used @@ -2280,21 +2024,11 @@ function mapNestedQueries( i < optimizedQuery.join.length && i < originalQuery.join.length; i++ ) { - const optimizedJoin = optimizedQuery.join[i]! - const originalJoin = originalQuery.join[i]! - - if ( - optimizedJoin.from.type === `queryRef` && - originalJoin.from.type === `queryRef` - ) { - queryMapping.set(optimizedJoin.from.query, originalJoin.from.query) - // Recursively map nested queries in joins - mapNestedQueries( - optimizedJoin.from.query, - originalJoin.from.query, - queryMapping, - ) - } + mapNestedFromQueries( + optimizedQuery.join[i]!.from, + originalQuery.join[i]!.from, + queryMapping, + ) } } } @@ -2306,10 +2040,6 @@ function getAllSources(query: QueryIR): Array { ] } -function getFirstFromAlias(from: QueryIR[`from`]): string { - return getFromSources(from)[0]?.alias ?? `` -} - function findProjectedIncludePaths( select: Record, targetPath: Array, @@ -2370,24 +2100,9 @@ function findProjectedIncludePaths( } if (value instanceof ConditionalSelect) { - const previousBranchGuards: Array = [] - for (const branch of value.branches) { - visitSelectValue(branch.value, path, [ - ...guards, - ...previousBranchGuards, - { condition: branch.condition, expected: true }, - ]) - previousBranchGuards.push({ - condition: branch.condition, - expected: false, - }) - } - if (value.defaultValue !== undefined) { - visitSelectValue(value.defaultValue, path, [ - ...guards, - ...previousBranchGuards, - ]) - } + visitConditionalSelect(value, guards, (child, childGuards) => + visitSelectValue(child, path, childGuards), + ) return } @@ -2469,60 +2184,33 @@ function extractIncludesFromSelect(select: Record): Array<{ }> = [] for (const [key, value] of Object.entries(select)) { if (key.startsWith(`__SPREAD_SENTINEL__`)) continue - if (value instanceof IncludesSubquery) { - results.push({ - key: getIncludesRoutingKey([key], results), - path: [key], - subquery: value, - guards: [], - }) - } else if (value instanceof ConditionalSelect) { - collectIncludesFromConditionalSelect(value, [key], [], results) - } else if (isNestedSelectObject(value)) { - // Check nested objects for IncludesSubquery — not supported yet + if (isNestedSelectObject(value)) { + // Plain nested includes remain invalid; conditional branches may contain them. assertNoNestedIncludes(value, key) + } else { + collectIncludesFromSelectValue(value, [key], [], results) } } return results } -function collectIncludesFromConditionalSelect( +/** Each branch requires every earlier branch to be false. */ +function visitConditionalSelect( conditional: ConditionalSelect, - prefixPath: Array, guards: Array, - results: Array<{ - key: string - path: Array - subquery: IncludesSubquery - guards: Array - }>, + visit: (value: any, guards: Array) => void, ): void { const previousBranchGuards: Array = [] for (const branch of conditional.branches) { - collectIncludesFromSelectValue( - branch.value, - prefixPath, - [ - ...guards, - ...previousBranchGuards, - { condition: branch.condition, expected: true }, - ], - results, - ) - previousBranchGuards.push({ - condition: branch.condition, - expected: false, - }) - } - - if (conditional.defaultValue !== undefined) { - collectIncludesFromSelectValue( - conditional.defaultValue, - prefixPath, - [...guards, ...previousBranchGuards], - results, - ) + visit(branch.value, [ + ...guards, + ...previousBranchGuards, + { condition: branch.condition, expected: true }, + ]) + previousBranchGuards.push({ condition: branch.condition, expected: false }) } + if (conditional.defaultValue !== undefined) + visit(conditional.defaultValue, [...guards, ...previousBranchGuards]) } function collectIncludesFromSelectValue( @@ -2537,13 +2225,15 @@ function collectIncludesFromSelectValue( }>, ): void { if (value instanceof IncludesSubquery) { - const key = getIncludesRoutingKey(prefixPath, results) + const key = getUniqueIncludesRoutingKey(prefixPath.join(`.`), results) results.push({ key, path: prefixPath, subquery: value, guards }) return } if (value instanceof ConditionalSelect) { - collectIncludesFromConditionalSelect(value, prefixPath, guards, results) + visitConditionalSelect(value, guards, (child, childGuards) => + collectIncludesFromSelectValue(child, prefixPath, childGuards, results), + ) return } @@ -2557,13 +2247,6 @@ function collectIncludesFromSelectValue( } } -function getIncludesRoutingKey( - path: Array, - entries: Array<{ key: string }>, -): string { - return getUniqueIncludesRoutingKey(path.join(`.`), entries) -} - function getUniqueIncludesRoutingKey( baseKey: string, entries: Array<{ key?: string; fieldName?: string }>, @@ -2609,70 +2292,31 @@ function assertNoNestedIncludes( /** * Replaces the IncludesSubquery at `path` inside a select value, through * nested objects and conditional branches, with a null Value placeholder, so - * processSelect() never sees it. Returns the new value and whether anything - * was replaced, so each level rebuilds only the objects that changed. + * processSelect() never sees it. Unchanged values retain their identity, so + * each level rebuilds only the objects that changed. */ -function replaceIncludesInSelectValue( - value: any, - path: Array, -): { value: any; replaced: boolean } { - if (path.length === 0 && value instanceof IncludesSubquery) { - return { value: new ValClass(null), replaced: true } - } - +function replaceIncludesInSelectValue(value: any, path: Array): any { + if (path.length === 0 && value instanceof IncludesSubquery) + return new ValClass(null) if (value instanceof ConditionalSelect) { - return replaceIncludesInConditionalSelect(value, path) - } - - if (path.length === 0 || !isNestedSelectObject(value)) { - return { value, replaced: false } - } - + const branches = value.branches.map((branch) => { + const next = replaceIncludesInSelectValue(branch.value, path) + return Object.is(next, branch.value) ? branch : { ...branch, value: next } + }) + const defaultValue = + value.defaultValue === undefined + ? undefined + : replaceIncludesInSelectValue(value.defaultValue, path) + return branches.some((branch, index) => branch !== value.branches[index]) || + !Object.is(defaultValue, value.defaultValue) + ? new ConditionalSelect(branches, defaultValue) + : value + } + if (path.length === 0 || !isNestedSelectObject(value)) return value const [head, ...rest] = path - const result = replaceIncludesInSelectValue(value[head!], rest) - if (!result.replaced) { - return { value, replaced: false } - } - return { - value: { - ...value, - [head!]: result.value, - }, - replaced: true, - } -} - -function replaceIncludesInConditionalSelect( - conditional: ConditionalSelect, - path: Array, -): { value: ConditionalSelect; replaced: boolean } { - let replaced = false - const branches = conditional.branches.map((branch) => { - const result = replaceIncludesInSelectValue(branch.value, path) - if (!result.replaced) { - return branch - } - replaced = true - return { ...branch, value: result.value } - }) - - let defaultValue = conditional.defaultValue - if (conditional.defaultValue !== undefined) { - const result = replaceIncludesInSelectValue(conditional.defaultValue, path) - if (result.replaced) { - replaced = true - defaultValue = result.value - } - } - - if (!replaced) { - return { value: conditional, replaced: false } - } - - return { - value: new ConditionalSelect(branches, defaultValue), - replaced: true, - } + const child = value[head!] + const next = replaceIncludesInSelectValue(child, rest) + return Object.is(next, child) ? value : { ...value, [head!]: next } } /** diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 619e4f7b2e..494ab28604 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -12,16 +12,15 @@ import { InvalidJoinConditionSameSourceError, InvalidJoinConditionSourceMismatchError, JoinCollectionNotFoundError, + UnsupportedFromTypeError, UnsupportedJoinSourceTypeError, UnsupportedJoinTypeError, } from '../../errors.js' -import { - getParentContextIdentity, - getParentContextValue, -} from '../equality-value-identity.js' +import { getParentContextIdentity } from '../equality-value-identity.js' import { ensureIndexForField } from '../../indexes/auto-index.js' import { validateJoinConditions } from '../join-conditions.js' import { + Value as ValClass, collectPropRefs, getFromSources, getPropRefSourceAlias, @@ -33,18 +32,14 @@ import { crossJoinParentRoutes } from './parent-routes.js' import { queriesMatchForCaching } from './query-equivalence.js' import { CALLBACK_INPUT, - INCLUDES_PUBLIC_KEY, attachRouteMetadata, attachRouteMetadataToResult, getCallbackInput, getNamespacedRouteMetadata, - getRouteMetadata, - getRoutedScalarMetadata, - stripRouteMetadata, + wrapInputRow, } from './route-metadata.js' import type { ValueIdentity } from '../equality-value-identity.js' -import type { CompileQueryFn } from './index.js' -import type { OrderByOptimizationInfo } from './order-by.js' +import type { CompilationResult, IncludesCompilationResult } from './index.js' import type { BasicExpression, CollectionSourceRef, @@ -62,8 +57,12 @@ import type { NamespacedAndKeyedStream, NamespacedRow, } from '../../types.js' -import type { QueryCache, QueryMapping, WindowOptions } from './types.js' -import type { CollectionSubscription } from '../../collection/subscription.js' +import type { QueryMapping } from './types.js' + +type CompileSubquery = ( + query: QueryIR, + parentKeyStream?: KeyedStream, +) => CompilationResult export type LazyDemandPlan = { id: string @@ -114,48 +113,6 @@ function parameterizeJoinInputByParentRoutes( ) } -function wrapJoinedInputRow( - alias: string, - row: any, - callbackValue?: unknown, -): NamespacedRow { - const finish = (namespaced: NamespacedRow): NamespacedRow => { - if (callbackValue !== undefined && callbackValue !== row) - Object.assign(namespaced, { - [CALLBACK_INPUT]: { [alias]: callbackValue }, - }) - return namespaced - } - const scalar = getRoutedScalarMetadata(row) - if (scalar) { - const namespaced = attachRouteMetadata( - { - ...getParentContextValue(scalar.parentContext), - [alias]: scalar.value, - [INCLUDES_PUBLIC_KEY]: scalar.publicKey, - }, - scalar.correlationKey, - scalar.parentContext, - ) as unknown as NamespacedRow - return finish(namespaced) - } - - if (row == null || typeof row !== `object`) { - return finish({ [alias]: row } as NamespacedRow) - } - - const route = getRouteMetadata(row) - const cleanRow = route ? stripRouteMetadata(row) : row - const namespaced: NamespacedRow = { - ...getParentContextValue(route?.parentContext), - [alias]: cleanRow, - } - if (route) { - attachRouteMetadata(namespaced, route.correlationKey, route.parentContext) - } - return finish(namespaced) -} - function getRouteJoinKey( row: NamespacedRow, source: string, @@ -233,17 +190,13 @@ export function processJoins( mainCollectionId: string, mainSource: string, allInputs: Record, - cache: QueryCache, queryMapping: QueryMapping, collections: Record, - subscriptions: Record, callbacks: Record, lazySources: Set, - optimizableOrderByCollections: Record, - setWindowFn: (windowFn: (options: WindowOptions) => void) => void, rawQuery: QueryIR, localFrom: From, - onCompileSubquery: CompileQueryFn, + onCompileSubquery: CompileSubquery, aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, @@ -251,308 +204,253 @@ export function processJoins( valueIdentity: ValueIdentity, parentKeyStream?: KeyedStream, ): NamespacedAndKeyedStream { - let resultPipeline = pipeline const localBindings = new Set( getFromSources(localFrom).map((source) => source.bindingId), ) - for (const joinClause of joinClauses) { - resultPipeline = processJoin( - resultPipeline, - joinClause, - sources, - mainCollectionId, - mainSource, + // The loop owns the source bindings and compiler context for each join. + function processJoin( + inputPipeline: NamespacedAndKeyedStream, + joinClause: JoinClause, + ): NamespacedAndKeyedStream { + const joinedSource = joinClause.from.alias + const mainAliases = Object.keys(sources) + const role = (ref: PropRef): JoinRefRole => { + const alias = getJoinRefAlias(ref) + if (ref.bindingId !== undefined) { + if (ref.bindingId === joinClause.from.bindingId) return `joined` + return localBindings.has(ref.bindingId) ? `main` : `ancestor` + } + if (alias === joinedSource) return `joined` + if ( + alias && + (mainAliases.includes(alias) || rawQuery.from.type === `unionAll`) + ) + return `main` + return `unknown` + } + const conditions = validateJoinConditions(joinClause.on).map( + ([left, right]) => + analyzeJoinExpressions(left, right, joinedSource, role), + ) + // The first equality supplies candidate demand; the full tuple decides matches. + const { mainExpr, joinedExpr } = conditions[0]! + const joinedExpressionUsesParent = conditions.some( + ({ joinedExpr: expression }) => + collectPropRefs(expression).some((ref) => { + const sourceRole = role(ref) + return sourceRole === `ancestor` || sourceRole === `unknown` + }), + ) + const routeJoinedSource = + parentKeyStream !== undefined && + (joinClause.from.type === `queryRef` || + ((joinClause.type === `right` || joinClause.type === `full`) && + !mainSourceIsParentFiltered) || + joinedExpressionUsesParent) + + // Get the joined source alias and input stream + const { + alias: processedJoinedSource, + input: joinedInput, + collectionId: joinedCollectionId, + } = processSource( + joinClause.from, allInputs, - cache, queryMapping, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - rawQuery, onCompileSubquery, aliasToCollectionId, aliasRemapping, sourceWhereClauses, - mainSourceIsParentFiltered, valueIdentity, - localBindings, - parentKeyStream, + `join`, + routeJoinedSource ? parentKeyStream : undefined, ) - localBindings.add(joinClause.from.bindingId) - } - return resultPipeline -} - -/** - * Processes a single join clause with lazy loading optimization. - * For LEFT/RIGHT/INNER joins, marks one side as "lazy" (loads on-demand based on join keys). - */ -function processJoin( - pipeline: NamespacedAndKeyedStream, - joinClause: JoinClause, - sources: Record, - mainCollectionId: string, - mainSource: string, - allInputs: Record, - cache: QueryCache, - queryMapping: QueryMapping, - collections: Record, - subscriptions: Record, - callbacks: Record, - lazySources: Set, - optimizableOrderByCollections: Record, - setWindowFn: (windowFn: (options: WindowOptions) => void) => void, - rawQuery: QueryIR, - onCompileSubquery: CompileQueryFn, - aliasToCollectionId: Record, - aliasRemapping: Record, - sourceWhereClauses: Map>, - mainSourceIsParentFiltered: boolean, - valueIdentity: ValueIdentity, - localBindings: ReadonlySet, - parentKeyStream?: KeyedStream, -): NamespacedAndKeyedStream { - const isCollectionRef = joinClause.from.type === `collectionRef` - - const joinedSource = joinClause.from.alias - const mainAliases = Object.keys(sources) - const role = (ref: PropRef): JoinRefRole => { - const alias = getJoinRefAlias(ref) - if (ref.bindingId !== undefined) { - if (ref.bindingId === joinClause.from.bindingId) return `joined` - return localBindings.has(ref.bindingId) ? `main` : `ancestor` + if (processedJoinedSource !== joinedSource) { + throw new InvalidJoinConditionSourceMismatchError() } - if (alias === joinedSource) return `joined` - if ( - alias && - (mainAliases.includes(alias) || rawQuery.from.type === `unionAll`) - ) - return `main` - return `unknown` - } - const conditions = validateJoinConditions(joinClause.on).map( - ([left, right]) => analyzeJoinExpressions(left, right, joinedSource, role), - ) - // The first equality supplies candidate demand; the full tuple decides matches. - const { mainExpr, joinedExpr } = conditions[0]! - const joinedExpressionUsesParent = conditions.some( - ({ joinedExpr: expression }) => - collectPropRefs(expression).some((ref) => { - const sourceRole = role(ref) - return sourceRole === `ancestor` || sourceRole === `unknown` - }), - ) - const routeJoinedSource = - parentKeyStream !== undefined && - (joinClause.from.type === `queryRef` || - ((joinClause.type === `right` || joinClause.type === `full`) && - !mainSourceIsParentFiltered) || - joinedExpressionUsesParent) - - // Get the joined source alias and input stream - const { - alias: processedJoinedSource, - input: joinedInput, - collectionId: joinedCollectionId, - } = processJoinSource( - joinClause.from, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - onCompileSubquery, - aliasToCollectionId, - aliasRemapping, - sourceWhereClauses, - valueIdentity, - routeJoinedSource ? parentKeyStream : undefined, - ) - - if (processedJoinedSource !== joinedSource) { - throw new InvalidJoinConditionSourceMismatchError() - } - // Add the joined source to the sources map - sources[joinedSource] = joinedInput - if (isCollectionRef) { - // Only direct collection references form new alias bindings. Subquery - // aliases reuse the mapping returned from the recursive compilation above. - aliasToCollectionId[joinedSource] = joinedCollectionId - } + // Add the joined source to the sources map + sources[joinedSource] = joinedInput - const mainCollection = collections[mainCollectionId] - const joinedCollection = collections[joinedCollectionId] + const mainCollection = collections[mainCollectionId] + const joinedCollection = collections[joinedCollectionId] - if (!mainCollection) { - throw new JoinCollectionNotFoundError(mainCollectionId) - } + if (!mainCollection) { + throw new JoinCollectionNotFoundError(mainCollectionId) + } - if (!joinedCollection) { - throw new JoinCollectionNotFoundError(joinedCollectionId) - } + if (!joinedCollection) { + throw new JoinCollectionNotFoundError(joinedCollectionId) + } - const sourceActivity = getActiveAndLazySources( - joinClause.type, - mainCollection, - joinedCollection, - mainSourceIsParentFiltered, - ) - const activeSource = routeJoinedSource - ? undefined - : sourceActivity.activeSource - const lazySource = sourceActivity.lazySource - - // Pre-compile the join expressions - const compiledMainExpressions = conditions.map((condition) => - compileExpression(condition.mainExpr), - ) - const compiledJoinedExpressions = conditions.map((condition) => - compileExpression(condition.joinedExpr), - ) + const sourceActivity = getActiveAndLazySources( + joinClause.type, + mainCollection, + joinedCollection, + mainSourceIsParentFiltered, + ) + const activeSource = routeJoinedSource + ? undefined + : sourceActivity.activeSource + const lazySource = sourceActivity.lazySource + + // Pre-compile the join expressions + const compiledMainExpressions = conditions.map((condition) => + compileExpression(condition.mainExpr), + ) + const compiledJoinedExpressions = conditions.map((condition) => + compileExpression(condition.joinedExpr), + ) - // Prepare both sides with the same key and raw-demand rules. - let mainPipeline = pipeline.pipe( - map(([currentKey, namespacedRow]) => - keyJoinInput( - currentKey, - namespacedRow, - mainSource, - `main`, - compiledMainExpressions, - routeJoinedSource, - valueIdentity, + // Prepare both sides with the same key and raw-demand rules. + let mainPipeline = inputPipeline.pipe( + map(([currentKey, namespacedRow]) => + keyJoinInput( + currentKey, + namespacedRow, + mainSource, + `main`, + compiledMainExpressions, + routeJoinedSource, + valueIdentity, + ), ), - ), - ) + ) - let joinedPipeline = joinedInput.pipe( - map(([currentKey, row, callbackValue]: [...InputRow, unknown?]) => - keyJoinInput( - currentKey, - wrapJoinedInputRow(joinedSource, row, callbackValue), - joinedSource, - `joined`, - compiledJoinedExpressions, - routeJoinedSource, - valueIdentity, + let joinedPipeline = joinedInput.pipe( + map(([currentKey, row, callbackValue]: [...InputRow, unknown?]) => + keyJoinInput( + currentKey, + wrapInputRow(joinedSource, row, callbackValue), + joinedSource, + `joined`, + compiledJoinedExpressions, + routeJoinedSource, + valueIdentity, + ), ), - ), - ) - - // Apply the join operation - if (![`inner`, `left`, `right`, `full`].includes(joinClause.type)) { - throw new UnsupportedJoinTypeError(joinClause.type) - } - - if (activeSource) { - // If the lazy collection comes from a subquery that has a limit and/or an offset clause - // then we need to deoptimize the join because we don't know which rows are in the result set - // since we simply lookup matching keys in the index but the index contains all rows - // (not just the ones that pass the limit and offset clauses) - const lazyFrom = activeSource === `main` ? joinClause.from : rawQuery.from - const limitedSubquery = - lazyFrom.type === `queryRef` && - (lazyFrom.query.limit !== undefined || - lazyFrom.query.offset !== undefined || - lazyFrom.query.singleResult) - const resultUnionLazySide = lazyFrom.type === `unionAll` - - const lazySourceJoinExpr = activeSource === `main` ? joinedExpr : mainExpr - const lazyAlias = activeSource === `main` ? joinedSource : mainSource - const lazyTargets = resultUnionLazySide - ? [] - : getLazyLoadTargets( - rawQuery, - lazyFrom, - lazyAlias, - lazySourceJoinExpr, - lazySource, - aliasRemapping, - ) + ) - if (!limitedSubquery && lazyTargets.length > 0) { - // This join can be optimized by having the active collection - // dynamically load keys into the lazy collection - // based on the value of the joinKey and by looking up - // matching rows in the index of the lazy collection - - // Mark the lazy source aliases as lazy - // this Set is passed by the liveQueryCollection to the compiler - // such that the liveQueryCollection can check it after compilation - // to know which source aliases should load data lazily (not initially) - for (const target of lazyTargets) { - lazySources.add(target.sourceId) - } + // Apply the join operation + if (![`inner`, `left`, `right`, `full`].includes(joinClause.type)) { + throw new UnsupportedJoinTypeError(joinClause.type) + } - const demandPlans = lazyTargets.map((target) => - registerLazyDemandPlan(callbacks, target), - ) - const demandWeights = new Map() + if (activeSource) { + // If the lazy collection comes from a subquery that has a limit and/or an offset clause + // then we need to deoptimize the join because we don't know which rows are in the result set + // since we simply lookup matching keys in the index but the index contains all rows + // (not just the ones that pass the limit and offset clauses) + const lazyFrom = activeSource === `main` ? joinClause.from : rawQuery.from + const limitedSubquery = + lazyFrom.type === `queryRef` && + (lazyFrom.query.limit !== undefined || + lazyFrom.query.offset !== undefined || + lazyFrom.query.singleResult) + const resultUnionLazySide = lazyFrom.type === `unionAll` + + const lazySourceJoinExpr = activeSource === `main` ? joinedExpr : mainExpr + const lazyAlias = activeSource === `main` ? joinedSource : mainSource + const lazyTargets = resultUnionLazySide + ? [] + : getLazyLoadTargets( + rawQuery, + lazyFrom, + lazyAlias, + lazySourceJoinExpr, + lazySource, + aliasRemapping, + ) - const activePipeline = - activeSource === `main` ? mainPipeline : joinedPipeline + if (!limitedSubquery && lazyTargets.length > 0) { + // This join can be optimized by having the active collection + // dynamically load keys into the lazy collection + // based on the value of the joinKey and by looking up + // matching rows in the index of the lazy collection + + // Mark the lazy source aliases as lazy + // this Set is passed by the liveQueryCollection to the compiler + // such that the liveQueryCollection can check it after compilation + // to know which source aliases should load data lazily (not initially) + for (const target of lazyTargets) { + lazySources.add(target.sourceId) + } - for (const target of lazyTargets) { - const fieldName = target.path[0] - if (fieldName) { - ensureIndexForField(fieldName, target.path, target.collection) + const demandPlans = lazyTargets.map((target) => + registerLazyDemandPlan(callbacks, target), + ) + const demandWeights = new Map< + string, + { key: unknown; weight: number } + >() + + const activePipeline = + activeSource === `main` ? mainPipeline : joinedPipeline + + for (const target of lazyTargets) { + const fieldName = target.path[0] + if (fieldName) { + ensureIndexForField(fieldName, target.path, target.collection) + } } - } - // Set up lazy loading: intercept active side's stream and dynamically load - // matching rows from lazy side based on join keys. - const activePipelineWithLoading: IStreamBuilder< - [key: string, value: JoinInputValue] - > = activePipeline.pipe( - tap((data) => { - for (const [[joinKey, [, , joinValue]], weight] of data.getInner()) { - if (joinValue == null) continue - const previous = demandWeights.get(joinKey) - const nextWeight = (previous?.weight ?? 0) + weight - if (nextWeight === 0) { - demandWeights.delete(joinKey) - } else { - demandWeights.set(joinKey, { - key: previous?.key ?? joinValue, - weight: nextWeight, - }) + // Set up lazy loading: intercept active side's stream and dynamically load + // matching rows from lazy side based on join keys. + const activePipelineWithLoading: IStreamBuilder< + [key: string, value: JoinInputValue] + > = activePipeline.pipe( + tap((data) => { + for (const [ + [joinKey, [, , joinValue]], + weight, + ] of data.getInner()) { + if (joinValue == null) continue + const previous = demandWeights.get(joinKey) + const nextWeight = (previous?.weight ?? 0) + weight + if (nextWeight === 0) { + demandWeights.delete(joinKey) + } else { + demandWeights.set(joinKey, { + key: previous?.key ?? joinValue, + weight: nextWeight, + }) + } } - } - const keys = new Set( - [...demandWeights.values()] - .filter(({ weight }) => weight > 0) - .map(({ key }) => key), - ) - for (let index = 0; index < lazyTargets.length; index++) { - const target = lazyTargets[index]! - callbacks[target.sourceId]?.setDemand?.(demandPlans[index]!, keys) - } - }), - ) + const keys = new Set( + [...demandWeights.values()] + .filter(({ weight }) => weight > 0) + .map(({ key }) => key), + ) + for (let index = 0; index < lazyTargets.length; index++) { + const target = lazyTargets[index]! + callbacks[target.sourceId]?.setDemand?.(demandPlans[index]!, keys) + } + }), + ) - if (activeSource === `main`) { - mainPipeline = activePipelineWithLoading - } else { - joinedPipeline = activePipelineWithLoading + if (activeSource === `main`) { + mainPipeline = activePipelineWithLoading + } else { + joinedPipeline = activePipelineWithLoading + } } } + + return mainPipeline.pipe( + joinOperator(joinedPipeline, joinClause.type as JoinType), + processJoinResults(joinedSource, mainAliases), + ) } - return mainPipeline.pipe( - joinOperator(joinedPipeline, joinClause.type as JoinType), - processJoinResults(joinedSource, mainAliases), - ) + for (const joinClause of joinClauses) { + pipeline = processJoin(pipeline, joinClause) + localBindings.add(joinClause.from.bindingId) + } + + return pipeline } type JoinRefRole = `joined` | `main` | `ancestor` | `unknown` @@ -628,26 +526,25 @@ function analyzeJoinExpressions( } /** - * Processes the join source (collection or sub-query) + * Compiles a FROM or JOIN source and propagates its source metadata. */ -function processJoinSource( +export function processSource( from: CollectionSourceRef | QueryRef, allInputs: Record, - collections: Record, - subscriptions: Record, - callbacks: Record, - lazySources: Set, - optimizableOrderByCollections: Record, - setWindowFn: (windowFn: (options: WindowOptions) => void) => void, - cache: QueryCache, queryMapping: QueryMapping, - onCompileSubquery: CompileQueryFn, + onCompileSubquery: CompileSubquery, aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, valueIdentity: ValueIdentity, + sourceKind: `from` | `join`, parentKeyStream?: KeyedStream, -): { alias: string; input: KeyedStream; collectionId: string } { +): { + alias: string + input: KeyedStream + collectionId: string + includes?: Array +} { switch (from.type) { case `descriptorRef`: return requireCollectionSource(from) @@ -663,13 +560,14 @@ function processJoinSource( aliasToCollectionId[from.alias] = from.collection.id return { alias: from.alias, - input: parentKeyStream - ? parameterizeJoinInputByParentRoutes( - input, - parentKeyStream, - valueIdentity, - ) - : input, + input: + sourceKind === `join` && parentKeyStream + ? parameterizeJoinInputByParentRoutes( + input, + parentKeyStream, + valueIdentity, + ) + : input, collectionId: from.collection.id, } } @@ -682,67 +580,27 @@ function processJoinSource( originalQuery && queriesMatchForCaching(from.query, originalQuery) ? originalQuery : from.query - const subQueryResult = onCompileSubquery( - queryToCompile, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - parentKeyStream, - ) + const subQueryResult = onCompileSubquery(queryToCompile, parentKeyStream) - // Pull up alias mappings from subquery to parent scope. - // This includes both the innermost alias-to-collection mappings AND - // any existing remappings from nested subquery levels. Object.assign(aliasToCollectionId, subQueryResult.aliasToCollectionId) Object.assign(aliasRemapping, subQueryResult.aliasRemapping) - - // Pull up source WHERE clauses from subquery to parent scope. - // This enables loadSubset to receive the correct where clauses for subquery collections. - // - // IMPORTANT: Skip pull-up for optimizer-created subqueries. These are detected when: - // 1. The outer alias (from.alias) matches the inner alias (from.query.from.alias) - // 2. The subquery was found in queryMapping (it's a user-defined subquery, not optimizer-created) - // - // For optimizer-created subqueries, the parent already has the sourceWhereClauses - // extracted from the original raw query, so pulling up would be redundant. - const isUserDefinedSubquery = queryMapping.has(from.query) - const fromInnerAlias = getFirstFromAlias(from.query) - const isOptimizerCreated = - !isUserDefinedSubquery && - fromInnerAlias !== undefined && - from.alias === fromInnerAlias - - if (!isOptimizerCreated) { - for (const [alias, whereClause] of subQueryResult.sourceWhereClauses) { - sourceWhereClauses.set(alias, whereClause) - } + // Optimizer-created wrappers already contributed their predicates to the + // parent. User-authored subqueries must export their own source predicates. + if ( + queryMapping.has(from.query) || + from.alias !== getFromSources(from.query.from)[0]?.alias + ) { + for (const [alias, where] of subQueryResult.sourceWhereClauses) + sourceWhereClauses.set(alias, where) } - - // Create a flattened remapping from outer alias to innermost alias. - // For nested subqueries, this ensures one-hop lookups (not recursive chains). - // - // Example with 3-level nesting: - // Inner: .from({ user: usersCollection }) - // Middle: .from({ activeUser: innerSubquery }) → creates: activeUser → user - // Outer: .join({ author: middleSubquery }, ...) → creates: author → user (not author → activeUser) - // - // We search through the PULLED-UP aliasToCollectionId (which contains the - // innermost 'user' alias), so we always map directly to the deepest level. - // This means aliasRemapping[lazyAlias] is always a single lookup, never recursive. + // Resolve directly to the innermost alias, so nested scopes need one lookup. const innerAlias = Object.keys(subQueryResult.aliasToCollectionId).find( (alias) => subQueryResult.aliasToCollectionId[alias] === subQueryResult.collectionId, ) - if (innerAlias && innerAlias !== from.alias) { + if (innerAlias && innerAlias !== from.alias) aliasRemapping[from.alias] = innerAlias - } // Extract the pipeline from the compilation result const subQueryInput = subQueryResult.pipeline @@ -762,13 +620,15 @@ function processJoinSource( callbackValue, ], ] = data - if (!parentKeyStream) { + if (sourceKind === `join` && !parentKeyStream) { return [key, value, callbackValue] as unknown as InputRow } return [ key, attachRouteMetadataToResult( - value, + sourceKind === `from` && value instanceof ValClass + ? value.value + : value, correlationKey, parentContext, publicKey, @@ -782,10 +642,13 @@ function processJoinSource( alias: from.alias, input: extractedInput as KeyedStream, collectionId: subQueryResult.collectionId, + includes: subQueryResult.includes, } } default: - throw new UnsupportedJoinSourceTypeError((from as any).type) + throw sourceKind === `join` + ? new UnsupportedJoinSourceTypeError((from as any).type) + : new UnsupportedFromTypeError((from as any).type) } } @@ -797,10 +660,6 @@ function encodeNonFiniteKey(_: string, value: unknown): unknown { : value } -function getFirstFromAlias(query: QueryIR): string | undefined { - return getFromSources(query.from)[0]?.alias -} - function processJoinResults( joinedSource: string, mainAliases: ReadonlyArray, @@ -818,24 +677,22 @@ function processJoinResults( const joinedKey = joined?.[0] const joinedNamespacedRow = joined?.[1] - // A joined row carries inherited parent aliases. When the main row is - // present it owns that scope; copy only the joined source from its row. - const mergedNamespacedRow: NamespacedRow = Object.assign( - {}, - mainNamespacedRow ?? joinedNamespacedRow, - ) - if (main == null) { - // The joined row may carry parent context with a local main alias. - // An absent main side must not turn that ancestor into a child row. - for (const alias of mainAliases) delete mergedNamespacedRow[alias] + // The main side owns inherited parent aliases. Missing local sides + // must not expose an ancestor with the same alias. + const mergeRows = ( + mainRow: NamespacedRow | undefined, + joinedRow: NamespacedRow | undefined, + ): NamespacedRow => { + const row = { ...(mainRow ?? joinedRow) } as NamespacedRow + if (main == null) for (const alias of mainAliases) delete row[alias] + if (joinedRow) row[joinedSource] = joinedRow[joinedSource]! + else delete row[joinedSource] + return row } - if (joinedNamespacedRow) { - mergedNamespacedRow[joinedSource] = joinedNamespacedRow[joinedSource]! - } else { - // A missing joined side cannot inherit a same-named parent source. - delete mergedNamespacedRow[joinedSource] - } - + const mergedNamespacedRow = mergeRows( + mainNamespacedRow, + joinedNamespacedRow, + ) const mainView = mainNamespacedRow && getCallbackInput(mainNamespacedRow) const joinedView = @@ -844,11 +701,9 @@ function processJoinResults( mainView !== mainNamespacedRow || joinedView !== joinedNamespacedRow ) { - const view = { ...(mainView ?? joinedView) } as NamespacedRow - if (main == null) for (const alias of mainAliases) delete view[alias] - if (joinedView) view[joinedSource] = joinedView[joinedSource]! - else delete view[joinedSource] - Object.assign(mergedNamespacedRow, { [CALLBACK_INPUT]: view }) + Object.assign(mergedNamespacedRow, { + [CALLBACK_INPUT]: mergeRows(mainView, joinedView), + }) } // Combine the main and joined keys without ambiguity: keys may contain diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index 5fa0591b4b..01c61fbf48 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -147,7 +147,7 @@ export function processOrderBy( return { compiledExpression: compileExpression(clauseWithoutAggregates), - compareOptions: buildCompareOptions(clause, collection), + compare: makeComparator(buildCompareOptions(clause, collection)), } }) // The row holds source aliases and any projected aggregate under $selected. @@ -160,30 +160,19 @@ export function processOrderBy( : null } - // Compare order terms in declaration order, stopping at the first difference. - const compare = (a: unknown, b: unknown) => { - if (orderByClause.length > 1) { - const arrayA = a as Array - const arrayB = b as Array - for (let i = 0; i < orderByClause.length; i++) { - const clause = compiledOrderBy[i]! - const compareFn = makeComparator(clause.compareOptions) - const result = compareFn(arrayA[i], arrayB[i]) - if (result !== 0) { - return result + // Compile comparators once; each row comparison reads the same options. + const compare = + compiledOrderBy.length <= 1 + ? (compiledOrderBy[0]?.compare ?? defaultComparator) + : (a: unknown, b: unknown) => { + const arrayA = a as Array + const arrayB = b as Array + for (let i = 0; i < compiledOrderBy.length; i++) { + const result = compiledOrderBy[i]!.compare(arrayA[i], arrayB[i]) + if (result !== 0) return result + } + return arrayA.length - arrayB.length } - } - return arrayA.length - arrayB.length - } - - if (orderByClause.length === 1) { - const clause = compiledOrderBy[0]! - const compareFn = makeComparator(clause.compareOptions) - return compareFn(a, b) - } - - return defaultComparator(a, b) - } let setSizeCallback: SetSizeCallback | undefined @@ -340,7 +329,6 @@ export function processOrderBy( // Each top-K owns its own window. Acquisition must cover all windows, // including moves made through a previously compiled root's callback. // Keep these reads live: setWindowFn updates the individual info objects. - const sameOrder = previous && deepEquals(previous.orderBy, info.orderBy) optimizableOrderByCollections[orderBySourceId] = previous ? { ...info, @@ -358,16 +346,14 @@ export function processOrderBy( Math.max( previous.offset + previous.limit, info.offset + info.limit, - ) - Math.min(previous.offset, info.offset) - ) - }, - get requiresFullSource() { - return ( - !sameOrder || - previous.requiresFullSource || - info.requiresFullSource + ) - this.offset ) }, + // Plan shape is fixed at compilation; only window bounds move. + requiresFullSource: + !deepEquals(previous.orderBy, info.orderBy) || + previous.requiresFullSource || + info.requiresFullSource, dataNeeded: (boundary) => Math.max( previous.dataNeeded?.(boundary) ?? 0, @@ -400,59 +386,31 @@ export function processOrderBy( } } - // Use grouped ordering when a groupKeyFn is provided (includes with limit/offset), - // otherwise use the standard global ordering operator. - if (groupKeyFn) { - return pipeline.pipe( - groupedOrderByWithFractionalIndex(valueExtractor, { - limit, - offset, - comparator: compare, - setSizeCallback, - groupKeyFn, - setWindowFn: ( - windowFn: (options: { offset?: number; limit?: number }) => void, - ) => { - setWindowFn((options) => { - windowFn(options) - if (orderByOptimizationInfo) { - orderByOptimizationInfo.offset = - options.offset ?? orderByOptimizationInfo.offset - orderByOptimizationInfo.limit = - options.limit ?? orderByOptimizationInfo.limit - } - }) - }, - }), - ) + const options = { + limit, + offset, + comparator: compare, + setSizeCallback, + setWindowFn: (windowFn: (options: WindowOptions) => void) => { + // Keep demand calculations aligned with the operator's current window. + setWindowFn((window) => { + windowFn(window) + if (orderByOptimizationInfo) { + orderByOptimizationInfo.offset = + window.offset ?? orderByOptimizationInfo.offset + orderByOptimizationInfo.limit = + window.limit ?? orderByOptimizationInfo.limit + } + }) + }, } - - // Use fractional indexing and return the tuple [value, index] return pipeline.pipe( - orderByWithFractionalIndex(valueExtractor, { - limit, - offset, - comparator: compare, - setSizeCallback, - setWindowFn: ( - windowFn: (options: { offset?: number; limit?: number }) => void, - ) => { - setWindowFn( - // We wrap the move function such that we update the orderByOptimizationInfo - // because that is used by the `dataNeeded` callback to determine if we need to load more data - (options) => { - windowFn(options) - if (orderByOptimizationInfo) { - orderByOptimizationInfo.offset = - options.offset ?? orderByOptimizationInfo.offset - orderByOptimizationInfo.limit = - options.limit ?? orderByOptimizationInfo.limit - } - }, - ) - }, - }), - // orderByWithFractionalIndex returns [key, [value, index]] - we keep this format + groupKeyFn + ? groupedOrderByWithFractionalIndex(valueExtractor, { + ...options, + groupKeyFn, + }) + : orderByWithFractionalIndex(valueExtractor, options), ) } diff --git a/packages/db/src/query/compiler/route-metadata.ts b/packages/db/src/query/compiler/route-metadata.ts index c3627659c5..25a9f0a710 100644 --- a/packages/db/src/query/compiler/route-metadata.ts +++ b/packages/db/src/query/compiler/route-metadata.ts @@ -1,4 +1,6 @@ import { isPlainObject } from '../../utils/type-guards.js' +import { getParentContextValue } from '../equality-value-identity.js' +import type { NamespacedRow } from '../../types.js' const ROUTED_SCALAR_VALUE = Symbol(`tanstack_db_routed_scalar_value`) export const CALLBACK_INPUT = Symbol(`tanstack_db_callback_input`) @@ -23,9 +25,15 @@ type RoutedResult = { [INCLUDES_PUBLIC_KEY]: unknown } -type PublicContainerProperty = { - descriptor: PropertyDescriptor - value?: { original: unknown; replacement: unknown } +type PublicContainer = { + value: object + properties: Record< + PropertyKey, + PropertyDescriptor & { child?: PublicContainer } + > + parents: Array + dirty?: true + copy?: object } export type RouteMetadata = { @@ -93,19 +101,13 @@ export function attachRouteMetadataToResult( return value } - if (isPlainObject(value)) { - return { - ...value, + return Object.assign( + isPlainObject(value) ? { ...value } : { [ROUTED_SCALAR_VALUE]: value }, + { [ROUTE_METADATA]: { correlationKey, parentContext }, [INCLUDES_PUBLIC_KEY]: publicKey, - } - } - - return { - [ROUTED_SCALAR_VALUE]: value, - [ROUTE_METADATA]: { correlationKey, parentContext }, - [INCLUDES_PUBLIC_KEY]: publicKey, - } satisfies RoutedResult + }, + ) } export function getRoutedScalarMetadata( @@ -120,15 +122,38 @@ export function getRoutedScalarMetadata( } const routed = value as RoutedResult - const route = routed[ROUTE_METADATA] return { value: routed[ROUTED_SCALAR_VALUE], - correlationKey: route.correlationKey, - parentContext: route.parentContext, + ...routed[ROUTE_METADATA], publicKey: routed[INCLUDES_PUBLIC_KEY], } } +/** Give FROM and JOIN rows the same route and callback namespace. */ +export function wrapInputRow( + alias: string, + row: unknown, + callbackValue?: unknown, +): NamespacedRow { + const scalar = getRoutedScalarMetadata(row) + const route = scalar ?? getRouteMetadata(row) + const namespaced = { + ...getParentContextValue(route?.parentContext), + [alias]: scalar + ? scalar.value + : route + ? stripRouteMetadata(row as Record) + : row, + } as NamespacedRow + if (route) + attachRouteMetadata(namespaced, route.correlationKey, route.parentContext) + if (scalar) + Object.assign(namespaced, { [INCLUDES_PUBLIC_KEY]: scalar.publicKey }) + if (callbackValue !== undefined && callbackValue !== row) + Object.assign(namespaced, { [CALLBACK_INPUT]: { [alias]: callbackValue } }) + return namespaced +} + /** A namespace keeps authored values beside its callback source overrides. * A one-element tuple denotes a flat union view: absent fields must disappear, * whereas named overrides replace only their own source aliases. @@ -167,80 +192,60 @@ export function transformPublicContainers( if (!Object.is(rootReplacement, value)) return rootReplacement if (!isPublicContainer(value)) return value - const parents = new WeakMap>() - const properties = new WeakMap< - object, - Map - >() - const dirty = new Set() - const visit = (current: object): void => { - if (properties.has(current)) return - const currentProperties = new Map() - properties.set(current, currentProperties) - for (const key of Reflect.ownKeys(current)) { + const containers = new Map() + const queue: Array = [] + const markDirty = (container: PublicContainer) => { + if (container.dirty) return + container.dirty = true + queue.push(container) + } + const visit = (current: object): PublicContainer => { + const existing = containers.get(current) + if (existing) return existing + const container: PublicContainer = { + value: current, + properties: Object.getOwnPropertyDescriptors(current), + parents: [], + } + containers.set(current, container) + for (const key of Reflect.ownKeys(container.properties)) { if (omittedKeys.has(key)) { - dirty.add(current) + delete container.properties[key] + markDirty(container) continue } - const descriptor = Object.getOwnPropertyDescriptor(current, key) - if (!descriptor) continue - const property: PublicContainerProperty = { descriptor } - currentProperties.set(key, property) + const descriptor = container.properties[key]! if (!descriptor.enumerable || !(`value` in descriptor)) continue const child = descriptor.value - const replacement = transformLeaf(child) - property.value = { original: child, replacement } - if (!Object.is(replacement, child)) { - dirty.add(current) - continue + descriptor.value = transformLeaf(child) + if (!Object.is(descriptor.value, child)) markDirty(container) + else if (isPublicContainer(child)) { + descriptor.child = visit(child) + descriptor.child.parents.push(container) } - if (!isPublicContainer(child)) continue - const childParents = parents.get(child) ?? new Set() - childParents.add(current) - parents.set(child, childParents) - visit(child) } + return container } - visit(value) - - const queue = [...dirty] - for (const current of queue) { - for (const parent of parents.get(current) ?? []) { - if (dirty.has(parent)) continue - dirty.add(parent) - queue.push(parent) - } + const root = visit(value) + for (const container of queue) { + for (const parent of container.parents) markDirty(parent) } - if (!dirty.has(value)) return value - - const copies = new WeakMap() - const copy = (current: object): object => { - if (!dirty.has(current)) return current - const existing = copies.get(current) - if (existing) return existing - - const result = Array.isArray(current) + const copy = (container: PublicContainer): object => { + if (!container.dirty) return container.value + if (container.copy) return container.copy + const result = Array.isArray(container.value) ? [] - : Object.create(Object.getPrototypeOf(current)) - copies.set(current, result) - for (const [key, property] of properties.get(current) ?? []) { - const descriptor = { ...property.descriptor } - if (property.value) { - const { original, replacement } = property.value - descriptor.value = !Object.is(replacement, original) - ? replacement - : isPublicContainer(original) - ? copy(original) - : original - } - Object.defineProperty(result, key, descriptor) + : Object.create(Object.getPrototypeOf(container.value)) + container.copy = result + for (const key of Reflect.ownKeys(container.properties)) { + const descriptor = container.properties[key]! + if (descriptor.child) descriptor.value = copy(descriptor.child) } - return result + return Object.defineProperties(result, container.properties) } - - return copy(value) + return copy(root) } -function isPublicContainer(value: unknown): value is object { +export function isPublicContainer(value: unknown): value is object { return Array.isArray(value) || isPlainObject(value) } From 5af7624937288cbe212f19ff23e8ba83caabb74e Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 16:16:47 -0600 Subject: [PATCH 12/19] refactor(db): simplify correlation handling --- .../oracle-reviews/callback-metadata.md | 46 +++++++++++++++++++ packages/db/src/query/compiler/index.ts | 14 ++---- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 9ae2c48ca4..a2e0a933ca 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -971,3 +971,49 @@ bindings supersede earlier bindings for these four refactored files only. | `packages/db/src/query/compiler/joins.ts` | `a5618bb1da3bff18b91fc4d9bf155447712e0bb1b68cd0f66a1cda0662d80450` | | `packages/db/src/query/compiler/order-by.ts` | `a104d4f4829e9b4ea801fcc60914772c1d97a47d7d46dbfa5e24f49724b12267` | | `packages/db/src/query/compiler/route-metadata.ts` | `bc6313f715e84949d0a6f8f5b48511bb3bc05ba5fd13a0b5e8a1033f3ee8e4df` | + + +### Integration with current main and final size + +The maintainer approved updating this branch from main. Merge `ce555e739` +incorporates `a2f346596c34faa7e65211134194337bd2e0007d`. The only conflict +was appended coverage-map prose. Both sides remain. This updates the branch +without merging PR #2312. + +The final compiler removes two redundant branches. `Object.is` already accepts +two `NaN` values, so its later `Number.isNaN` case cannot change the answer. +Object spread already accepts `undefined`, so parent-context copying needs no +separate empty-object case. Accepted laws, oracle grammars, comparisons and +budgets remain unchanged. + +The default `pnpm size:compare --packages db --files` now compares the integrated +tree with current main `a2f346596`, rather than the earlier merge-base. + +| Measurement | Main gzip | Final gzip | Gzip delta | Brotli delta | +| --- | ---: | ---: | ---: | ---: | +| Full `@tanstack/db` package | 109,658 B | 109,601 B | −57 B | −48 B | +| Typical app | 96,591 B | 96,529 B | −62 B | −31 B | + +Every measured scenario shrinks with gzip. The Effect scenario grows by one +byte with Brotli. The earlier original-base and exact-base diagnostics remain +historical measurements of their named trees. + +The frozen integrated source passes 6,533 tests across 16 files, including the +primary owners, restart publication, replay, distinct collisions and include +receivers. Focused no-emit TypeScript and ESLint pass, with zero lint warnings. +Evidence is `/private/tmp/callback-metadata/ci-monitor/size-final-integration`: +`size.log`, `green.*`, `types.log`, `eslint.log` and `final-fingerprints.json`. +The refreshed compiler index binding below supersedes its previous binding. +Other compiler sources retain the previous refactor bindings. + +`packages/db/src/query/compiler/index.ts`: `c356929cce3db647ec28f79ff2dd14bead69450ac6567176ec0bb63d6b8d970d`. + + +All seven controls reject the intended wrong behavior on the frozen integrated +source: HAVING view loss 551, over-normalized predicates 696, raw-only cache 26, +window overwrite 48, joined overwrite 24, joined last-only eight and joined +first-only eight. `size-mutant-summary.json` retains the assertion errors and +source hashes. The missing-dependency setup attempt remains separately in +`setup-failure-1`; it supplies no RED evidence. Existing dependency links +resolved that copied-tree setup. No install or production mutation occurred. +The isolated copies were restored and both trees match the frozen fingerprints. diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 9b16d78850..150a8c0e20 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -189,9 +189,9 @@ function projectParentContext( valueIdentity: ValueIdentity, ): Record { const inherited = getRouteMetadata(nsRow)?.parentContext - const inheritedValue = getParentContextValue(inherited) - const parentContext: Record = - inheritedValue === undefined ? {} : { ...inheritedValue } + const parentContext: Record = { + ...getParentContextValue(inherited), + } const projectedBindings = { ...getParentContextProjectedBindings(inherited), } @@ -281,13 +281,7 @@ function correlationValuesEqual(left: unknown, right: unknown): boolean { if (left == null || right == null) return false const normalizedLeft = normalizeValue(left) const normalizedRight = normalizeValue(right) - return ( - Object.is(normalizedLeft, normalizedRight) || - (typeof normalizedLeft === `number` && - typeof normalizedRight === `number` && - Number.isNaN(normalizedLeft) && - Number.isNaN(normalizedRight)) - ) + return Object.is(normalizedLeft, normalizedRight) } /** From bcdf5a19190a234ab04aa0c29857c87a9c5670ad Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 17:10:37 -0600 Subject: [PATCH 13/19] fix(db): retain new demand failures during shared replay --- .changeset/retain-loading-sibling-replay.md | 5 + docs/contributing/oracle-coverage.md | 20 +++ .../2026-10-30-release-replay-retention.md | 154 ++++++++++++++++++ .../oracle-reviews/callback-metadata.md | 19 +++ .../tests/multi-tab-electric-oracle.test.ts | 59 +++++-- packages/db/src/collection/subscription.ts | 2 +- packages/db/src/query/live/ARCHITECTURE.md | 7 +- .../query/live/collection-config-builder.ts | 6 +- ...ad-subset-replay-refinement-oracle.test.ts | 151 ++++++++++++++++- 9 files changed, 396 insertions(+), 27 deletions(-) create mode 100644 .changeset/retain-loading-sibling-replay.md create mode 100644 docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md diff --git a/.changeset/retain-loading-sibling-replay.md b/.changeset/retain-loading-sibling-replay.md new file mode 100644 index 0000000000..64368eee0d --- /dev/null +++ b/.changeset/retain-loading-sibling-replay.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Keep the last complete live-query result when a new include demand fails while a sibling subscription is still replaying after truncate. A successful sibling load cannot publish an incomplete replacement. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index f19a185a8a..af46459838 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6392,3 +6392,23 @@ useful extension: the hash identity oracle owns those semantic distinctions, but its helper-level evidence does not by itself exercise distinct storage. The separate Index structural-collision obligation above remains open. See the appended [review evidence](oracle-reviews/2026-10-30-distinct-collisions.md). + +## Loading sibling replay and final catch-up reach (2026-10-30) + +The primary load-subset replay owner now constructs 48 finite histories where +an include with no demand at truncate starts under its sibling's loading replay. +It measures held failure (44 histories), failure retirement (33), and later own +coverage (12), through the existing public retention and recovery comparisons. +The original guard fails the quiet retention check. Subscription-scoped verdict +and successful-coverage calibrations remain in that primary owner. + +The browser multi-tab Electric owner preserves its stop prefix and arbitrary +tail and constructs an abandoned reservation immediately before final catch-up. +This removes dependence on sampling that boundary without changing its model, +80-run budgets, comparisons or reach minima. + +See [the versioned experiment record](oracle-reviews/2026-10-30-release-replay-retention.md) +for authority, RED, reach, mutant, grammar controls, historical gate limits and +receiving-path limits. This is core Collection publication evidence and a Node +SQLite/coordinator receiving witness with controlled Electric streams. It does +not claim Effect, provider, framework, deeper include, or browser-process closure. diff --git a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md new file mode 100644 index 0000000000..447c10beb4 --- /dev/null +++ b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md @@ -0,0 +1,154 @@ +# Release replay retention and generated final catch-up + +The release workflow for `e75f8455a78bc33c43376288218e9912dc2a3f4c` +failed before publication. Its two failures have different causes. +The DB property exposed a production retention defect. The browser property +passed its behavior comparisons but missed a required state. Neither failure +establishes a regression in the callback-metadata size refactor: the DB +counterexample also fails on the exact release source. + +## Law and the first divergent observation + +Authority: live-query `ARCHITECTURE.md`, **Replay participants and failure**, +and normative law 9. A query's subscriptions to one source share publication. +Every acquisition started under that replay participates. A failed held demand +keeps the last complete result. Sibling success cannot decide its verdict. +Demand retirement or successful own replacement are the neighboring controls. + +The original `replay.ordered-failed-replay` counterexample uses seed +`-533188721`, path `428:3:3:6`. Its commands hold loads, truncate markers, +settle the two oldest loads successfully, fail the newest load, then remove +marker 1. The scenario has descending order, offset 1, limit 2, no ties or +peer, an anti-join and an inline include over on-demand markers. At quiet +loading, the expected retained snapshot is empty; production publishes row 1. + +The adapter receipts distinguish production failure from demand retirement. +The include held no demand at truncate. It later acquires marker key 1 while +the sibling join replay is loading. That include acquisition fails and remains +held and un-aborted. Its sibling later succeeds. Only cleanup unloads the +failed acquisition, after the mismatching observation. The old guard looked +only for a failed sibling replay. The repair also recognizes a loading replay +that still withholds publication. Ordinary failures outside replay retain their +existing request-error behavior. + +## Primary owner and experiment order + +`load-subset-replay-refinement-oracle.test.ts` remains the primary owner. +The change retains the subscription-scoped verdict and successful-coverage +corrections previously reviewed on PR 2312. Those corrections concern the +model, not a new production promise: shared publication does not merge replay +verdicts. Their two finite calibrations remain in the same owner. + +Before the runtime edit, a new constructive finite grammar produced the +intended RED on unchanged main. The failing assertion reports one measured new +sibling failure and the same empty-snapshot/row-1 mismatch. The first reach +recorder incorrectly excluded retired historical acquisitions; the corrected +pre-edit run measures acquisitions actually held at truncate. Both logs remain. + +The final grammar has 48 histories: two order directions, three widths, one or +two successful pre-failure settlements, and four later actions (none, marker +change, rank move, or another marker replay). Every history uses the existing +model, driver, retention comparison after each command, quiet checkpoint, and +final authoritative recovery. No expected row set was added to the histories. + +The initial repair trial passed all retention comparisons but failed the newly +declared own-coverage reach. The final recorder observes same-subscription key +coverage, rather than predicate object identity: a later demand may construct +new request data. The grammar also includes an explicit later replay control. +This post-trial reach work does not retroactively establish a pre-edit recovery +reach gate. The original retention RED and the receiving path were obtained +before production changed. + +Final measured reach: 44 histories with a first-demand failure under a loading +sibling; 33 with retirement of such failed demand; 12 with later successful +coverage by its own subscription. The three states belong to this constructive +grammar. Adding them to the shared result type initially made the unconstrained +campaign demand them by accident; the explicit owner assignment corrects that +without removing any pre-existing reach obligation. + +The original-guard mutant fails at the intended quiet retention comparison, +with the same one measured new sibling failure. It was restored in a `finally` +block and the repaired source hash was checked. Setup errors are recorded +separately and are not RED or mutant rejection evidence. + +## Browser grammar repair + +`multi-tab-electric-oracle.test.ts` owns the existing five browser laws. +The failed 80-history random campaign missed **reset-abandoned source recovered +during final catch-up**. Its fixed campaign reached that state four times. +A previous stop/peer prefix constructs other required states, but this state +still depended on an unconstrained 8–24-step tail. + +The repair keeps that prefix, tail, 80-run budgets, model, comparisons and reach +minimum. A suffix creates a fresh handle, recovers and catches up every tab, +delivers a new change to one tab without committing it, then resets its +reservation. The existing final delivery must observe the abandoned commit's +failure before restarting and checking convergence. Two/three tabs, source tab, +key and recovery method vary. Removing the partial delivery or reset loses the +premise; committing before final catch-up moves the failure to a different cut. +The original named final-delivery replays remain unchanged. + +## Review and limits + +This is an author self-review. The earlier independent diagnosis supplied the +receipt trace; it is not an independent review of the final implementation. + +- ORC-001/002/003/005/009: accepted architecture law, existing independent + row/retention model, literate grammar and real Collection sync boundary; + subscription identities and requested keys are adapter receipts. +- ORC-004/013/015: finite direction/width/settlement/change grammar reconstructs + the reported witness and adjacent controls. Absent replay is the ordinary + failure neighbor; obsolete or retired requests cannot count as held failures. + Browser suffix ablations lose its stated premise. Reach is measured. +- ORC-006/010: original guard rejected at the retention assertion; original + failure, initial reach-only failures and setup errors remain distinguishable. + Existing fixture cleanup remains in place. +- ORC-007: existing fixed/random properties and direct seed/path interface are + unchanged. The new 48-cell grammar is finite enumeration, not a random + property. Browser fixed/random campaigns retain the same 80-run grammar. +- ORC-008: no new semantic model state. New receipt fields serve reach only. + Retained subscription distinction is required because sibling success and + own success permit different verdicts. +- ORC-011: no new shared semantic fault hypothesis. Independent adapter trace + rejected the alternative retirement explanation of this failure. +- ORC-012/014/016: this record states the bounded claim and the unchanged + controlled premises. Histories use the shared comparison, with no unit-test + expected answers. + +Core evidence is limited to live-query Collection publication with one ordered +source, a one-level inline include, and the controlled core sync adapter. This +change does not claim Effect failure disposal, provider delivery, framework +scheduling, arbitrary joins or deeper recursion. Those paths have separate +owners; the modified builder failure path serves live-query Collections. +Browser evidence uses real SQLite, BroadcastChannel and coordinator code in one +Node process with controlled Electric streams. It does not establish installed +SDK/HTTP, real browser multi-process timing, on-demand row contents or existing +known-gap closure. No laws, assertions, run budgets or reach minima are weakened. + +## Final commands, results and source binding + +Local runs use saved configs under `/private/tmp/callback-metadata/release-replay-repair` to resolve the exact worktree source with cached dependencies. Every command sets `pnpm_config_verify_deps_before_run=false`; Vitest uses one worker. + +| Check | Command or selector | Result | +| --- | --- | --- | +| Pre-edit primary RED | `pnpm exec vitest run --config /vitest.config.mjs -t 'keeps new include failures behind a loading sibling replay'` | Intended quiet retention mismatch; measured first-demand sibling failure 1. | +| Final finite grammar | Same command | PASS, 48 histories, failure/retirement/own-recovery reach 44/33/12. | +| Original guard mutant | Same command with only the old guard restored | Intended quiet retention mismatch, measured failure 1. Repaired source restored and hash verified. | +| Full primary owner | `pnpm exec vitest run --config /vitest.config.mjs` | 43 passed, two accidental new-reach obligations failed. After explicit ownership correction, both affected fixed/random campaigns passed. All 45 tests pass across those runs. | +| Core neighbors | Saved `neighbors.config.mjs` | 21 tests in three files pass, covering replay refinement, failure boundaries and request errors. | +| Browser fixed/random campaigns | `pnpm exec vitest run --config /browser.config.mjs -t 'keeps reserved commits lawful'` | Both pass. Final catch-up recovery reached 80/80 histories in each campaign. | +| Browser final-delivery controls | Same config with the two existing final-delivery selectors | Both existing shared-model replays pass. | +| Focused TypeScript | `pnpm exec tsc --noEmit -p /db-types.json` and `browser-types.json` | Both pass. | + +The browser campaign retains 80 runs for each seed mode. Its expected injected stream errors are diagnostics, not comparison failures. The first two browser setup failures, the initial reach-only failures, and the intermediate TypeScript literal-inference error remain separate from RED/GREEN evidence. + +The following SHA256 fingerprints bind the main-based executable evidence to PR 2318 commit `0e5a5234b9fe4a26244a0e52d3862916a7f4ce7e`. The callback branch port has separate source fingerprints and results in [its evidence record](callback-metadata.md#release-blocking-loading-sibling-replay-port-2026-10-30). + +| File | SHA256 | +| --- | --- | +| `packages/db/src/query/live/collection-config-builder.ts` | `bb1b62aaf00aed1dc5eaa5e537671d92bb99def7a53a6e951ef38f73b00f5088` | +| `packages/db/src/collection/subscription.ts` | `bd171bdc97b38a79ebbc81d7fb605267787308559ad4c0646dff1e072a43798a` | +| `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts` | `e5ce4a77fe8d16f4b0d26f4c71fdadb53b2779c8e7e8313a7eb32d2de3ebdd38` | +| `packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts` | `67c16d284522417f1a77e0a36efcc552b718f858ea8ec00771f001f001160782` | + +Focused ESLint reports zero errors and two existing `no-shadow` warnings. Prettier, whitespace, and `pnpm lint:changed` pass. No run budget decreased or disappeared. diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index a2e0a933ca..590c8d83ff 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -1017,3 +1017,22 @@ source hashes. The missing-dependency setup attempt remains separately in `setup-failure-1`; it supplies no RED evidence. Existing dependency links resolved that copied-tree setup. No install or production mutation occurred. The isolated copies were restored and both trees match the frozen fingerprints. + +## Release-blocking loading sibling replay port (2026-10-30) + +The release workflow for main `e75f8455a78bc33c43376288218e9912dc2a3f4c` exposed a retention counterexample that also reproduces on callback head `5af7624937288cbe212f19ff23e8ba83caabb74e`. Independent adapter receipts show that a new include demand failed while a sibling replay was still loading. The include had no acquisition at truncate, and its failed demand remained held. This is an enforcement gap in architecture law 9, not evidence of a regression in the compiler removals. + +[PR 2318](https://github.com/TanStack/db/pull/2318) carries the main-based repair and its [versioned experiment record](2026-10-30-release-replay-retention.md). The same primary extension ran on unchanged callback production first: `callback-primary-red.json` fails at quiet retention with measured sibling failure 1. The one-condition guard then changed from failed replacement to pending replacement. `callback-primary-green.json` passes all 48 finite histories with reach 44/33/12. The full primary owner passes all 45 tests on this branch. The old guard's pre-edit RED and the main-based original-guard mutant preserve the intended violated comparison. + +The SQLite grammar port is byte-identical to the main-based fix. Its two 80-history campaigns and two existing final-delivery controls pass there. This branch's focused TypeScript, ESLint, formatting, whitespace and `pnpm lint:changed` pass. ESLint retains two existing warnings. All 14 compiler source fingerprints from the final size integration remain unchanged. The bounded core/controlled-host claims and historical recovery-reach gate limitation remain as stated in the experiment record. + +After the guard port, `pnpm_config_verify_deps_before_run=false pnpm size:compare --packages db --files` reports package gzip 109658 to 109595 bytes (-63), Brotli -47, against merge-base `a2f346596`. Typical app gzip changes from 96591 to 96527 (-64), with Brotli +21. Every measured gzip scenario shrinks. This does not claim every Brotli scenario shrinks. The later release merge changes versions and changelogs rather than these runtime sources. + +Scratch evidence is under `/private/tmp/callback-metadata/release-replay-repair`. These SHA256 fingerprints bind the callback port separately from the main-based repair: + +| File | SHA256 | +| --- | --- | +| `packages/db/src/query/live/collection-config-builder.ts` | `5ef5db6bde45a6a34aa54706772041653996bd1c3e974d0dcb7104884f12cfb0` | +| `packages/db/src/collection/subscription.ts` | `bd171bdc97b38a79ebbc81d7fb605267787308559ad4c0646dff1e072a43798a` | +| `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts` | `e5ce4a77fe8d16f4b0d26f4c71fdadb53b2779c8e7e8313a7eb32d2de3ebdd38` | +| `packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts` | `67c16d284522417f1a77e0a36efcc552b718f858ea8ec00771f001f001160782` | diff --git a/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts index f109c45c08..341715569a 100644 --- a/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts @@ -989,24 +989,53 @@ describe(`multi-tab Electric over one persisted database`, () => { }), steps: fc.array(historyStep, { minLength: 8, maxLength: 24 }), }) - .map(({ tabCount, source, key, recovery, stopOrder, steps }) => { - const tab = source % tabCount - const peer = (tab + 1) % tabCount - return { + .map( + ({ tabCount, - steps: [ - ...stopOrder.flatMap((pick): Array => [ + source, + key, + recovery, + stopOrder, + steps, + }): GeneratedHistory => { + const tab = source % tabCount + const peer = (tab + 1) % tabCount + return { + tabCount, + steps: [ + ...stopOrder.flatMap((pick): Array => [ + { kind: `server change`, key, pick: 0 }, + { kind: `advance`, tab, pick: 0 }, + { kind: `stop`, tab, pick }, + { kind: `catch up`, tab: peer, pick: 0 }, + { kind: `recover`, tab, pick: recovery }, + { kind: `catch up`, tab, pick: 0 }, + ]), + ...steps, + // Final catch-up must encounter an abandoned reservation in every + // generated history. Start a fresh handle, recover every tab, and + // deliver one new change without committing it. Reset then abandons + // that reservation. No later history step commits or recovers it; + // the existing final-delivery comparison must observe its failure + // before the existing recovery establishes convergence (Laws 2/5). + // The source tab, key and two/three-tab host vary with this grammar. + // Removing advance or reset loses that premise; catching up here + // would move the failure out of the final observation cut. + { kind: `deploy` }, + ...Array.from( + { length: tabCount }, + (_, index): Array => [ + { kind: `recover`, tab: index, pick: recovery }, + { kind: `catch up`, tab: index, pick: 0 }, + ], + ).flat(), { kind: `server change`, key, pick: 0 }, { kind: `advance`, tab, pick: 0 }, - { kind: `stop`, tab, pick }, - { kind: `catch up`, tab: peer, pick: 0 }, - { kind: `recover`, tab, pick: recovery }, - { kind: `catch up`, tab, pick: 0 }, - ]), - ...steps, - ], - } - }) + { kind: `peer reset` }, + ], + } + }, + ) type GeneratedHistory = { tabCount: number; steps: Array } type ServerLog = { diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 428dd2fac8..949e0eb4d4 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -871,7 +871,7 @@ export class CollectionSubscription /** * A lazy demand that failed outside this subscription's replay, while a - * failed replay of another of the graph's subscriptions to this collection + * replay of another of the graph's subscriptions to this collection still * holds publication, fails this subscription's replacement too. It keeps * the last publication until every demand load that joins this replay * loads, a later replay succeeds, or this subscription's last demand diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 3f25b55484..158d40dcf6 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1562,10 +1562,11 @@ A live query's subscriptions to one collection, such as a join and an include of the same source, share one publication barrier. None of their replays publishes, and none whose last demand retired closes, while another of them still has replay work, so demand that any of them starts meanwhile joins its -own open replay. While a failed replay of any of them holds publication, a +own open replay. While a replay of any of them still holds publication, a lazy demand of any of them that fails after initial-query readiness fails its -own subscription's replacement too, so retiring the other replay's failed -demand cannot publish the window without this demand's rows. It holds until +own subscription's replacement too. This includes a subscription that held no +demand at truncate. Settling a sibling's replay or retiring its failed demand +cannot publish the window without this demand's rows. It holds until every demand load that joins it loads, a later replay of the source succeeds, or the subscription's last demand retires. The query retries the failed demand at once and the retry joins the hold, so the hold ends only after the failed diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 47687c4c07..26acc639d8 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -596,8 +596,8 @@ export class CollectionConfigBuilder< if (this.liveQueryCollection?._lifecycle.status === `ready`) { // The failure settles the demand: the request errors of its keys show // it, and ordered work must not wait for a load that already ended. - // Output held for this demand publishes now, unless a failed replay of - // this source holds publication: the failure then holds this + // Output held for this demand publishes now, unless a replay of this + // source still holds publication: the failure then holds this // subscription's publication too, so retiring the other replay's // failed demand cannot publish without this demand's rows. demand.settled = true @@ -605,7 +605,7 @@ export class CollectionConfigBuilder< this.collectionSources.some( (source) => source.collection === collection && - this.subscriptions[source.sourceId]?.hasFailedTruncateReplacement, + this.subscriptions[source.sourceId]?.hasPendingTruncateReplacement, ) ) subscription.failReplacement(normalized) diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts index 805eafe3af..6f94d276e3 100644 --- a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts @@ -320,6 +320,9 @@ type OrderedReach = { ownReplayFailureComparisons: number siblingReplayFailureComparisons: number siblingSuccessfulCoverage: number + newSiblingReplayFailures: number + newSiblingReplayRetirements: number + newSiblingReplayRecoveries: number replayLoads: number abortRejections: number preloadChecks: number @@ -469,6 +472,9 @@ async function checkOrderedReplay( source?: BarrierSource // Publication is shared, but each subscription owns its replay verdict. subscription: LoadSubsetOptions[`subscription`] + // This subscription held no acquisition at truncate, then started demand + // during its sibling replay. Measured only from adapter receipts. + newSiblingReplay?: boolean keys?: Set ordered?: boolean } @@ -491,6 +497,8 @@ async function checkOrderedReplay( ambiguous: boolean pending: Map own: Set + // Acquisition receipts held at truncate, used only for declared reach. + subscriptionsAtTruncate: Set } const barriers = new Map() // Every barrier this history raised, in raise order. @@ -520,6 +528,18 @@ async function checkOrderedReplay( ambiguous: [...barriers.values()].some(({ ambiguous }) => ambiguous), pending: new Map(barriers.get(source)?.pending), own: new Set(), + subscriptionsAtTruncate: new Set( + [...acquired].flatMap(([options, count]) => + count > 0 && + loads.some( + (load) => + load.source === source && + load.subscription === options.subscription, + ) + ? [options.subscription] + : [], + ), + ), } barriers.set(source, barrier) raisedBarriers.push(barrier) @@ -568,6 +588,13 @@ async function checkOrderedReplay( barrier.own.has(load) && !options.signal?.aborted ) { + if ( + load.newSiblingReplay && + [...barrier.pending.values()].some( + (pending) => pending.subscription !== options.subscription, + ) + ) + reach.newSiblingReplayFailures++ barrier.failures.set(load, options) barrier.history.push(load) } @@ -597,8 +624,9 @@ async function checkOrderedReplay( failed.ordered === load.ordered && failed.keys !== undefined && [...failed.keys].every((key) => load.keys?.has(key)) - ) + ) { barrier.failures.delete(failed) + } } if (siblingFailures.some((failed) => !barrier.failures.has(failed))) replayScopeViolation ??= `successful sibling removed another subscription's replay failure` @@ -618,7 +646,11 @@ async function checkOrderedReplay( const failures = barrier.failures.size for (const owned of [barrier.pending, barrier.failures]) { for (const [load, held] of owned) { - if (held === options) owned.delete(load) + if (held === options) { + if (owned === barrier.failures && load.newSiblingReplay) + reach.newSiblingReplayRetirements++ + owned.delete(load) + } } } // One failed demand retires while another still holds the barrier. @@ -782,6 +814,19 @@ async function checkOrderedReplay( ) ) reach.successWhileSubscriptionOwes++ + // Later successful coverage by this subscription is a recovery control. + // Coverage by a sibling cannot decide this subscription's failure. + if ( + loads.some( + (earlier) => + earlier.newSiblingReplay && + earlier.outcome === `failed` && + earlier.source === source && + earlier.subscription === options.subscription && + [...(earlier.keys ?? [])].every((key) => keys.has(key)), + ) + ) + reach.newSiblingReplayRecoveries++ loadedKeys.push({ source, keys, at: ++requestClock }) } const recordRequestFailure = ( @@ -917,6 +962,9 @@ async function checkOrderedReplay( ownReplayFailureComparisons: 0, siblingReplayFailureComparisons: 0, siblingSuccessfulCoverage: 0, + newSiblingReplayFailures: 0, + newSiblingReplayRetirements: 0, + newSiblingReplayRecoveries: 0, replayLoads: 0, abortRejections: 0, preloadChecks: 0, @@ -975,6 +1023,14 @@ async function checkOrderedReplay( source: `markers`, keys: requestedKeys(`markers`, options), } + load.newSiblingReplay = + barriers.get(`markers`) !== undefined && + !barriers + .get(`markers`)! + .subscriptionsAtTruncate.has(options.subscription) && + [...(barriers.get(`markers`)?.pending.values() ?? [])].some( + (pending) => pending.subscription !== options.subscription, + ) loads.push(load) reach.markerLoads++ // The throw happens while the subscription registers the @@ -1024,6 +1080,14 @@ async function checkOrderedReplay( source: `markers`, keys: requestedKeys(`markers`, options), } + load.newSiblingReplay = + barriers.get(`markers`) !== undefined && + !barriers + .get(`markers`)! + .subscriptionsAtTruncate.has(options.subscription) && + [...(barriers.get(`markers`)?.pending.values() ?? [])].some( + (pending) => pending.subscription !== options.subscription, + ) loads.push(load) reach.markerLoads++ const participant = joinBarrier(`markers`, load, options) @@ -1495,9 +1559,10 @@ async function checkOrderedReplay( const raised = [...barriers.values()] if (raised.some(({ ambiguous }) => ambiguous)) return if (live.status === `error`) return - expect(JSON.parse(JSON.stringify(live.toArray)), label).toEqual( - JSON.parse(retainedRows), - ) + expect( + JSON.parse(JSON.stringify(live.toArray)), + `${label}; new sibling replay failures: ${reach.newSiblingReplayFailures}`, + ).toEqual(JSON.parse(retainedRows)) if (raised.some(({ failures }) => failures.size > 0)) { reach.retainedFailedReplays++ } else reach.heldPublicationChecks++ @@ -2485,6 +2550,75 @@ describe(`Ordered windows across truncate replays`, () => { ) }) + // A window with no initial include demand can activate its first include + // acquisition while a sibling join replay is still loading. Failure of that + // demand belongs to the shared publication barrier (law 9), even though its + // subscription had no replay of its own at the truncate. Adapter receipts + // prove both premises. This bounded grammar varies order, width, prior + // settlements and later demand changes; it reuses every ordinary observation + // and the independent retention account above. A move can retire the failed + // demand; a later truncate can replace it with successful own coverage. + it(`keeps new include failures behind a loading sibling replay`, async () => { + const states = [ + `newSiblingReplayFailures`, + `newSiblingReplayRetirements`, + `newSiblingReplayRecoveries`, + ] as const + const declared = createOracleReach(`replay.new-sibling-demand`, states) + for (const direction of [`desc`, `asc`] as const) { + for (const limit of [2, 1, 3] as const) { + for (const settles of [2, 1]) { + for (const change of [`mark`, `none`, `move`, `replay`] as const) { + declared.beginHistory() + const observed = await checkOrderedReplay( + [ + { type: `hold` }, + { type: `truncateMarkers` }, + ...Array.from({ length: settles }, () => ({ + type: `settle` as const, + ok: true, + keepHolding: true, + newest: false, + })), + { type: `settle`, ok: false, newest: true, keepHolding: true }, + ...(change === `none` + ? [] + : change === `replay` + ? [{ type: `truncateMarkers` as const }] + : change === `mark` + ? [{ type: `mark` as const, id: 1, on: false }] + : [ + { + type: `move` as const, + id: 1, + rank: direction === `desc` ? 60 : 1, + }, + ]), + ], + { + ...ordinaryOrderedScenario, + direction, + offset: 1, + limit, + joined: `none`, + includes: true, + markerMode: `on-demand`, + }, + ) + for (const state of states) { + if (observed[state] > 0) declared.reach(state) + } + } + } + } + } + console.info( + `replay.new-sibling-demand`, + Object.fromEntries(declared.counts()), + ) + declared.assertReached() + }, 120_000) + // This finite history grammar varies which source replays, the order of // two load failures, join form, direction and window width. Join plus include // gives sibling subscriptions; the root replay gives same-subscription @@ -2629,6 +2763,9 @@ describe(`Ordered windows across truncate replays`, () => { ownReplayFailureComparisons: 0, siblingReplayFailureComparisons: 0, siblingSuccessfulCoverage: 0, + newSiblingReplayFailures: 0, + newSiblingReplayRetirements: 0, + newSiblingReplayRecoveries: 0, replayLoads: 0, abortRejections: 0, preloadChecks: 0, @@ -2670,6 +2807,10 @@ describe(`Ordered windows across truncate replays`, () => { `ownReplayFailureComparisons`, `siblingReplayFailureComparisons`, `siblingSuccessfulCoverage`, + // The constructive new-sibling grammar declares these rare premises. + `newSiblingReplayFailures`, + `newSiblingReplayRetirements`, + `newSiblingReplayRecoveries`, `successWhileSubscriptionOwes`, ]) // Counts are histories, rather than repeated events in one trace. From 72c8ef1fea3d4726a0ff097a489d760ade1d8cb2 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 30 Oct 2026 17:32:09 -0600 Subject: [PATCH 14/19] Scope replay request-error oracle to each subscription --- docs/contributing/oracle-coverage.md | 14 +++++ .../2026-10-30-release-replay-retention.md | 63 +++++++++++++++++++ .../oracle-reviews/callback-metadata.md | 13 ++++ ...ad-subset-replay-refinement-oracle.test.ts | 45 ++++++++++--- 4 files changed, 128 insertions(+), 7 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index af46459838..1d10868379 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6412,3 +6412,17 @@ for authority, RED, reach, mutant, grammar controls, historical gate limits and receiving-path limits. This is core Collection publication evidence and a Node SQLite/coordinator receiving witness with controlled Electric streams. It does not claim Effect, provider, framework, deeper include, or browser-process closure. + + +### Subscription-scoped request-error assertion (2026-10-30) + +The primary load-subset replay owner now retains subscription identity in its +later-request and successful-coverage account. A sibling requesting overlapping +source keys cannot exempt the failed demand's public health error. Its existing +96-history sibling-success grammar measures the actual owed-error assertion in +four histories, alongside four successful-sibling-coverage histories. A targeted +omitted-error observation passes the former source-wide account and fails the +scoped health comparison. All existing comparisons, histories, budgets and +reach minima remain. The controlled core, one-level include and peer limits +remain those of this owner; no provider or framework claim follows. See the +[versioned repair record](oracle-reviews/2026-10-30-release-replay-retention.md#subscription-scoped-request-error-checker-repair). diff --git a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md index 447c10beb4..06e78d2e28 100644 --- a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md +++ b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md @@ -152,3 +152,66 @@ The following SHA256 fingerprints bind the main-based executable evidence to PR | `packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts` | `67c16d284522417f1a77e0a36efcc552b718f858ea8ec00771f001f001160782` | Focused ESLint reports zero errors and two existing `no-shadow` warnings. Prettier, whitespace, and `pnpm lint:changed` pass. No run budget decreased or disappeared. + +## Subscription-scoped request-error checker repair + +CodeRabbit review 5719714297 covered repair head +`0e5a5234b9fe4a26244a0e52d3862916a7f4ce7e`. Its one behavioral +finding identified a false-green request-error classification. Architecture +law 16 and the glossary's request-error rule already scope a lazy demand's +error to that demand. A sibling subscription asking for or loading the same +source keys cannot decide it. Shared publication does not merge verdicts. +The existing complete authoritative source-recovery control still decides the +held demands together; an individual sibling verdict does not supply it. + +The old account stored successful keys by source alone. Both its later-request +and later-coverage exemptions omitted subscription identity. The repair keeps +that identity in the recorded successful acquisition and requires it in both +exemptions. This is an oracle enforcement repair. Production is unchanged. + +The existing 96-history sibling-success grammar varies direction, join form, +width, replay source and settlement order. A receipt probe found four histories +with eight observation cuts where the old account exempted an error the scoped +account owed. Production listed the exact injected error at all eight cuts. +The cut concerns a later sibling request; the experiment did not establish a +later successful sibling-coverage cut with an otherwise owed request error. +An initial reach counter for that stronger premise correctly failed at zero. +It remains in the scratch evidence. The final declared reach names the actual +sibling-request error comparison and reaches four histories. The existing +successful-sibling-coverage reach also remains four. + +Calibration supplies a wrong public observation to the same health comparison: +omit the injected error only at those distinguishing cuts. The old account +accepts it. The scoped account rejects it at `after settle: request error`, +with expected true and actual false. A broader temporary production mutant +that suppressed every ready-query replay error already failed the old checker; +it is not evidence for this newly repaired gap. Both temporary sources were +restored. The real production owner passes all 45 tests, including fixed and +random campaigns, with the stronger classifier. The reported seeds and paths +from issues 2294 and 2315 each pass the direct replay interface as well. These +are bounded replay results, not closure of either whole issue or provider +adapter conformance from issue 2118. + +ORC review (author self-review): 001/002/003 use the accepted demand-scoped +law, independently evaluated request keys and explanatory local prose. +004/013 retain all 96 legal histories and their direction/join/width/source/ +settlement neighbors; collapsing subscription identity is the wrong rule. +005/006/015 exercise the real Collection sync path and public health error +identity after each command, measure four distinguishing histories and reject +the targeted omission. 007 retains all fixed/random campaigns, budgets and +seed/path replays. 008/009 retain subscription identity because equal source +keys can have different demand verdicts. 010 retains primary mismatch and +cleanup separation. 011 has no new shared production/model semantic-fault +hypothesis: the defect was an exemption before the assertion. 012 is this +versioned bounded record. 014 keeps the controlled core boundary and makes no +provider/framework handoff claim. 016 adds no unit expectation or standalone +fixture. No law, comparison, budget or prior reach minimum was weakened. + +The checker evidence is stored under +`/private/tmp/callback-metadata/release-replay-repair/review-5719714297`. +Commands use the saved `vitest.config.mjs` for the full primary owner and +`-t 'retains sibling replay failures through successful coverage'` for the +finite calibration, with one worker and dependency verification disabled. +The amended primary owner SHA256 is `a2ab454ff5a77aa47bf57cbd590b67a2a5602d3dd6897af051fb1819b9482ce7` on both branches. The earlier +table binds the original repair evidence. Its runtime and browser hashes remain +unchanged; callback runtime hashes remain bound by its separate port record. diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 590c8d83ff..1bec01c769 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -1036,3 +1036,16 @@ Scratch evidence is under `/private/tmp/callback-metadata/release-replay-repair` | `packages/db/src/collection/subscription.ts` | `bd171bdc97b38a79ebbc81d7fb605267787308559ad4c0646dff1e072a43798a` | | `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts` | `e5ce4a77fe8d16f4b0d26f4c71fdadb53b2779c8e7e8313a7eb32d2de3ebdd38` | | `packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts` | `67c16d284522417f1a77e0a36efcc552b718f858ea8ec00771f001f001160782` | + + +### Subscription-scoped request-error checker port (2026-10-30) + +Review 5719714297 on repair PR 2318 identified a primary-oracle exemption that +combined sibling subscriptions by source keys. The same unchanged classifier +was present here. The oracle-only repair retains subscription identity for +later requests and coverage, and measures the owed public health comparison in +the existing 96-history grammar. The shared +[versioned amendment](2026-10-30-release-replay-retention.md#subscription-scoped-request-error-checker-repair) +records the accepted law, old/new distinguishing probe, targeted omitted-error +calibration, intermediate zero-reach result and limits. This port changes no +runtime source, browser grammar, shipped-size result or compiler fingerprint. diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts index 6f94d276e3..7ad2fdfd13 100644 --- a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts @@ -320,6 +320,8 @@ type OrderedReach = { ownReplayFailureComparisons: number siblingReplayFailureComparisons: number siblingSuccessfulCoverage: number + // Owed request-error comparisons after another subscription asked for its keys. + siblingRequestErrorChecks: number newSiblingReplayFailures: number newSiblingReplayRetirements: number newSiblingReplayRecoveries: number @@ -749,18 +751,19 @@ async function checkOrderedReplay( // request error on the query's own node. The error stays until one of these // meets it: // - its demand retires, which aborts its request and unloads it; - // - a later load of the source asks for some of its keys: that newer + // - a later load of the same subscription asks for some of its keys: that newer // request of the demand decides them, and keys it no longer asks for // left the demand (ARCHITECTURE.md law 16: a lazy demand's error clears // when that demand settles or is released; a key's latest load decides // it); - // - a later load of the same source covers its keys; + // - a later load of the same subscription covers its keys; // - a replay of its source that rose after the failure completes, since a // replay asks again for every held demand. // A failure that joins a replay waits for its subscription's verdict, // which comes once that subscription's participants settled, so it owes // no error before then. The barrier that holds publication stays shared - // across the query's subscriptions to the source. Keys come from evaluating the request's + // across the query's subscriptions to the source. A sibling's successful + // load cannot decide this demand's error. Keys come from evaluating the request's // `where` over a small key domain, independent of production's bookkeeping. // The account combines the production concepts of a failed request and the // replay verdict that decides it; it names no production structure. @@ -790,6 +793,7 @@ async function checkOrderedReplay( const requestFailures: Array = [] const loadedKeys: Array<{ source: BarrierSource + subscription: LoadSubsetOptions[`subscription`] keys: Set at: number }> = [] @@ -827,7 +831,12 @@ async function checkOrderedReplay( ) ) reach.newSiblingReplayRecoveries++ - loadedKeys.push({ source, keys, at: ++requestClock }) + loadedKeys.push({ + source, + subscription: options.subscription, + keys, + at: ++requestClock, + }) } const recordRequestFailure = ( load: Load, @@ -868,6 +877,7 @@ async function checkOrderedReplay( (later, index) => index > at && later.source === failure.source && + later.subscription === failure.options.subscription && !later.ordered && [...failure.keys].some((key) => later.keys?.has(key)), ) @@ -875,6 +885,7 @@ async function checkOrderedReplay( const covered = loadedKeys.some( (loaded) => loaded.source === failure.source && + loaded.subscription === failure.options.subscription && loaded.at > failure.at && [...failure.keys].every((key) => loaded.keys.has(key)), ) @@ -962,6 +973,7 @@ async function checkOrderedReplay( ownReplayFailureComparisons: 0, siblingReplayFailureComparisons: 0, siblingSuccessfulCoverage: 0, + siblingRequestErrorChecks: 0, newSiblingReplayFailures: 0, newSiblingReplayRetirements: 0, newSiblingReplayRecoveries: 0, @@ -1442,6 +1454,19 @@ async function checkOrderedReplay( if (owed.length === 0) return reach.requestErrorChecks++ for (const failure of owed) { + // Reach is measured at the health assertion, after a sibling asked + // for overlapping keys that cannot decide this subscription's error. + if ( + loads.some( + (later, index) => + index > loads.indexOf(failure.load) && + later.source === failure.source && + later.subscription !== failure.options.subscription && + !later.ordered && + [...failure.keys].some((key) => later.keys?.has(key)), + ) + ) + reach.siblingRequestErrorChecks++ expect( [live, peer].some((query) => query?.health.error?.errors.some( @@ -2687,7 +2712,7 @@ describe(`Ordered windows across truncate replays`, () => { it(`retains sibling replay failures through successful coverage`, async () => { const declared = createOracleReach( `replay.subscription-success-scope`, - [`siblingSuccessfulCoverage`] as const, + [`siblingSuccessfulCoverage`, `siblingRequestErrorChecks`] as const, { minimum: 1 }, ) for (const direction of [`asc`, `desc`] as const) { @@ -2720,8 +2745,12 @@ describe(`Ordered windows across truncate replays`, () => { includes: true, }, ) - if (observed.siblingSuccessfulCoverage > 0) - declared.reach(`siblingSuccessfulCoverage`) + for (const state of [ + `siblingSuccessfulCoverage`, + `siblingRequestErrorChecks`, + ] as const) { + if (observed[state] > 0) declared.reach(state) + } } } } @@ -2763,6 +2792,7 @@ describe(`Ordered windows across truncate replays`, () => { ownReplayFailureComparisons: 0, siblingReplayFailureComparisons: 0, siblingSuccessfulCoverage: 0, + siblingRequestErrorChecks: 0, newSiblingReplayFailures: 0, newSiblingReplayRetirements: 0, newSiblingReplayRecoveries: 0, @@ -2807,6 +2837,7 @@ describe(`Ordered windows across truncate replays`, () => { `ownReplayFailureComparisons`, `siblingReplayFailureComparisons`, `siblingSuccessfulCoverage`, + `siblingRequestErrorChecks`, // The constructive new-sibling grammar declares these rare premises. `newSiblingReplayFailures`, `newSiblingReplayRetirements`, From a2e75c30402deb815abd01783ea816a2edfe3ea8 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 06:59:25 -0700 Subject: [PATCH 15/19] fix(solid-db): keep source row reads outside query dependencies Track query input preparation before resolving the live-query Collection outside Solid dependency tracking. Preserve reactive callback and config inputs while avoiding native-row-driven binding recreation. Law: result row reads cannot add query-input dependencies; real input signals still compile again. Authority: reactive query-identity contract and Solid row-materialization owner; versioned record in docs/contributing/oracle-reviews/callback-metadata.md. Reach:32 source-only-write histories and8 query-input-change histories, each with its own required checkpoint. RED: extended lifetime owner on fa5df11a fails8 native histories at the source-only subscription comparison,12 controls pass. GREEN:201 owner tests and336 Solid runtime tests pass; focused TS, ESLint, changed-file lint, formatting and whitespace pass. Mutants: original hook fails8 source-only comparisons; over-untracked hook fails all8 application-signal result comparisons. Limits: finite settled actual Solid host; local Vitest3.2.4 vs CI4.1.12; full pnpm test and builds not run. No other-framework claim. --- .changeset/fix-solid-query-input-tracking.md | 5 +++ docs/contributing/oracle-coverage.md | 2 + .../oracle-reviews/callback-metadata.md | 20 +++++++++ packages/solid-db/src/useLiveQuery.ts | 43 ++++++------------- .../tests/row-materialization-oracle.test.tsx | 35 ++++++++++++--- 5 files changed, 71 insertions(+), 34 deletions(-) create mode 100644 .changeset/fix-solid-query-input-tracking.md diff --git a/.changeset/fix-solid-query-input-tracking.md b/.changeset/fix-solid-query-input-tracking.md new file mode 100644 index 0000000000..76e3c33bfc --- /dev/null +++ b/.changeset/fix-solid-query-input-tracking.md @@ -0,0 +1,5 @@ +--- +'@tanstack/solid-db': patch +--- + +Prevent native source row reads during query compilation from recreating the Solid framework binding. Keep actual callback and config query parameters reactive. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 54073fafbe..b4944f2ec7 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6435,6 +6435,8 @@ prototype methods while native nodes/setters supply updates. Original-code, missing-node, accessor/method, and reordered-tail mutants distinguish these repairs. No output alias topology or public elapsed-time budget is imposed. +The callback integration adds eight signal-driven lifetime histories to the same Solid materialization owner: callback/config inputs cross one/two array levels and ordinary/native payloads. The original 48 input histories remain. The 32 identity/lifetime histories each enforce source-only-write reach; the eight added histories also enforce query-input-change reach and compare selected values plus rebinding. Application input preparation stays tracked, while initial compilation and Collection startup run untracked. The extended pre-production campaign rejects eight native lifetime cases on integration head `fa5df11a`; all 201 owner tests and 336 Solid runtime tests then pass locally. Restoring the old hook fails the source-only subscription comparison. Untracking query construction too fails all eight new signal-result comparisons. Existing comparisons and budgets remain. The [integration amendment](oracle-reviews/callback-metadata.md#solid-binding-lifetime-at-compilation-2026-11-02) records commands, runner differences, exact evidence, and limits. These finite settled-host cuts establish neither native-source-only value liveness nor other frameworks' dependency tracking. + The local-only draft driver does not claim top-level symbol-write admission. Its shared preservation comparison still checks every field actually admitted, including nested symbols and all actual mutation originals. The proxy-revert diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 1bec01c769..59b4a06f8e 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -1049,3 +1049,23 @@ the existing 96-history grammar. The shared records the accepted law, old/new distinguishing probe, targeted omitted-error calibration, intermediate zero-reach result and limits. This port changes no runtime source, browser grammar, shipped-size result or compiler fingerprint. + +### Solid binding lifetime at compilation (2026-11-02) + +CI for integration head `fa5df11a85342ad6fee373aedb3b7391e7b25630` exposed eight failures in Solid's existing row-materialization owner. Native-store row reads during initial compilation became dependencies of the framework binding's Collection memo. A later native-store-only write then created another live-query Collection. A diagnostic stack places these reads inside `resolveLiveQueryValue`, during synchronous initial graph drainage. This was a receiving-boundary integration defect, not a new metadata law. + +**Law and authority:** incidental source row reads must not become query-input dependencies. Actual application query-input signal reads must continue to compile the binding again. The authorities are the binding-lifetime contract at the start of `packages/solid-db/tests/row-materialization-oracle.test.tsx`, the Framework row snapshot isolation coverage entry, and the reactive query-identity law in `packages/db/tests/conformance/contract.ts`. + +**Repair:** Solid evaluates the public input and prepares callback/config query builders while tracking their application reads. It then resolves the prepared query within Solid's `untrack`. Initial graph work and supplied Collection startup therefore cannot capture row dependencies in the Collection memo. Preparing config query functions before that boundary preserves their query-input dependencies. The existing core adapter helper supplies preparation. This repair changes no core compiler source. + +**Reach and test gap:** the original owner already detected the native-row lifetime defect in CI. The callback branch had not run this real Solid receiver before integration. The owner now adds eight lifetime histories across callback/config, one/two array levels, and ordinary/native payloads. Each first checks the existing source-only-write comparison, then changes an application signal and checks both selected output and rebinding. The original 48 input histories remain, including supplied Collection controls. Across the owner, 32 identity/lifetime histories require their source-only-write cut, and the eight new histories additionally require `query-input-change`. Each history enforces its own minimum of one at each declared cut. Four new histories admit native payloads and four are ordinary controls. The complete owner grows from 193 to 201 tests without weakening comparisons or budgets. + +**RED before production:** the extended owner ran against unchanged `fa5df11a` production. The selected lifetime campaign failed eight histories at `row reads cannot rebind the hook` (expected one subscription start, observed two), with twelve passing controls and 181 unselected tests. Both callback and config inputs failed at each array depth. The separate earlier original-grammar run also reproduced the eight CI failures across identity/lifetime. Setup failures while configuring module resolution preceded these runs and supply no RED evidence. + +**GREEN:** the complete materialization owner passed all 201 tests. The wider Solid runtime suite passed 336 tests across eight files, including shared conformance, snapshot isolation, actual Solid host consumers, parameter changes, Suspense, and persisted on-demand readiness. Focused TypeScript and ESLint checks passed for both changed files. Local runs used cached Vitest 3.2.4 with actual Solid 1.9.11, browser conditions, and a single module realm. CI uses Vitest 4.1.12, so these runs do not substitute for the pushed head's CI result. No dependencies were installed. + +**Hostile mutants:** a scratch alias restoring the original hook failed eight selected lifetime histories at the source-only subscription comparison. A second scratch alias untracked query construction as well as compilation. All eight new signal histories rejected it at the public result comparison: `inputVersion` stayed zero after the signal became one. The latter demonstrates that suppressing every dependency cannot satisfy the expanded owner. Neither experiment changed the final worktree sources. + +**Replays and evidence:** scratch logs and task-local configs are under `/private/tmp/callback-metadata/solid-integration-20261102`. `pre-edit-binding.json` predates the production edit. `extended-parent-red.log`, `green.log`, `solid-suite-green.log`, `original-mutant.log`, and `over-untracked-mutant.log` retain the actual results. The focused runner commands use `pnpm_config_verify_deps_before_run=false pnpm exec vitest run --config ` with at most two workers. The RED and restored-hook campaigns select `lifetime`; the over-untracked campaign selects `reactive=true`. `config.mjs` runs the primary owner, `full.config.mjs` runs the eight runtime files, and the two mutant configs substitute only the hook under test. `tsconfig.json` supplies cached dependency paths for focused TypeScript and ESLint checks. These mappings are scratch artifacts and do not change package configuration. + +**Limits:** finite histories on the actual Solid host at its settled cut. The change does not specify native-source-only value liveness, arbitrary query callbacks that intentionally read live row data as parameters, every asynchronous source-start schedule, other framework adapters, or real browser rendering. Runtime suite counts exclude type-only test registration. Focused TypeScript passes separately. Whole-repository `pnpm test` and builds were not run. This is author self-review. External CodeRabbit coverage remains paused on an older commit and supplies no current-head review. diff --git a/packages/solid-db/src/useLiveQuery.ts b/packages/solid-db/src/useLiveQuery.ts index 164a941e0d..8023d19b87 100644 --- a/packages/solid-db/src/useLiveQuery.ts +++ b/packages/solid-db/src/useLiveQuery.ts @@ -5,6 +5,7 @@ import { createResource, createSignal, onCleanup, + untrack, } from 'solid-js' import { ReactiveMap } from '@solid-primitives/map' import { @@ -12,8 +13,8 @@ import { createLiveQueryObserver, getLiveQueryStatusFlags, getPublicCollection, - isCollection, isSingleResultCollection, + prepareLiveQueryValue, resolveLiveQueryValue, toBoundaryError, } from '@tanstack/db' @@ -254,34 +255,18 @@ export function useLiveQuery( ) { const collection = createMemo( () => { - if (configOrQueryOrCollection.length === 1) { - // This is a query function - check if it returns null/undefined - const queryBuilder = new BaseQueryBuilder() as InitialQueryBuilder - const result = configOrQueryOrCollection(queryBuilder) - - if (result === undefined || result === null) { - // Disabled query - return null - return null - } - - return resolveLiveQueryValue(result, { gcTime: DEFAULT_GC_TIME_MS }) - } - - const innerCollection = configOrQueryOrCollection() - - if (innerCollection === undefined || innerCollection === null) { - // Disabled query - return null - return null - } - - if (isCollection(innerCollection)) { - innerCollection.startSyncImmediate() - return innerCollection as Collection - } - - return resolveLiveQueryValue(innerCollection, { - gcTime: DEFAULT_GC_TIME_MS, - }) + const value = + configOrQueryOrCollection.length === 1 + ? configOrQueryOrCollection( + new BaseQueryBuilder() as InitialQueryBuilder, + ) + : configOrQueryOrCollection() + // Query-input reads define binding dependencies. Compilation can drain + // source rows synchronously; those reads belong to the result instead. + const prepared = prepareLiveQueryValue(value, undefined) + return untrack(() => + resolveLiveQueryValue(prepared, { gcTime: DEFAULT_GC_TIME_MS }), + ) }, undefined, { name: `TanstackDBCollectionMemo` }, diff --git a/packages/solid-db/tests/row-materialization-oracle.test.tsx b/packages/solid-db/tests/row-materialization-oracle.test.tsx index e02e6dd44f..8ed5e98c4c 100644 --- a/packages/solid-db/tests/row-materialization-oracle.test.tsx +++ b/packages/solid-db/tests/row-materialization-oracle.test.tsx @@ -14,6 +14,8 @@ * with same-parent controls. Resize histories grow, shrink and empty arrays. * The shared snapshot owner's immutable-update law also preserves source values. * All three inputs run on the real Solid host at one or two array levels. + * Signal-driven query inputs also change their selected value and must rebind. + * This distinguishes input tracking from incidental native row reads. * Comparison follows the host's settled cut. This law does not promise * identity for an element whose source object changed, live mutable-source * values, arbitrary getters, cyclic updates, or a wall-clock work bound. @@ -43,6 +45,7 @@ type History = { nested: boolean native: boolean law: `identity` | `lifetime` | `reorder` | `resize` + reactiveInput?: boolean nativeElements?: boolean initialParent?: boolean nextParent?: boolean @@ -61,8 +64,11 @@ function* histories(): Generator { `lifetime`, `reorder`, `resize`, - ] as const) + ] as const) { yield { input, nested, native, law } + if (input !== `collection` && law === `lifetime`) + yield { input, nested, native, law, reactiveInput: true } + } } // Replacing a parent can expose the same native child through raw descriptors @@ -135,7 +141,7 @@ async function runHistory(history: History) { ? `same-parent-control` : `parent-transition` const reach = createOracleReach( - `solid.materialization.${history.input}.${history.nested}.${history.native}.${history.law}.${history.initialParent}.${history.nextParent}`, + `solid.materialization.${history.input}.${history.nested}.${history.native}.${history.law}.${history.initialParent}.${history.nextParent}.${history.reactiveInput}`, history.law === `reorder` || history.law === `resize` ? [ history.law === `reorder` ? `source-reorder` : `array-resize`, @@ -148,12 +154,14 @@ async function runHistory(history: History) { `same-row-scalar-edit`, `element-replacement`, `source-only-write`, + ...(history.reactiveInput ? [`query-input-change`] : []), ], ) reach.beginHistory() const resources = new ScenarioLifetime() await resources.run(async () => { const [nativePayload, setNativePayload] = createStore({ value: 0 }) + const [inputVersion, setInputVersion] = createSignal(0) const initialValues = history.law === `reorder` ? [0, 10] : [0] const tags = initialValues.map((value) => history.nativeElements ? createStore({ value })[0] : { value }, @@ -190,8 +198,14 @@ async function runHistory(history: History) { resources.defer(() => disposeEffects?.()) const query = createRoot((dispose) => { resources.defer(dispose) - const build = (q: InitialQueryBuilder) => - q.from({ row: source }).select(({ row }) => ({ ...row })) + const build = (q: InitialQueryBuilder) => { + if (!history.reactiveInput) + return q.from({ row: source }).select(({ row }) => ({ ...row })) + const version = inputVersion() + return q + .from({ row: source }) + .select(({ row }) => ({ ...row, inputVersion: version })) + } return history.input === `collection` ? useLiveQuery(() => source) : history.input === `config` @@ -404,6 +418,17 @@ async function runHistory(history: History) { `row reads cannot rebind the hook`, ).toBe(starts) reach.reach(`source-only-write`) + if (history.reactiveInput) { + setInputVersion(1) + await waitFor(() => + expect(Reflect.get(current(), `inputVersion`)).toBe(inputVersion()), + ) + expect( + subscriptions.mock.calls.length, + `query-input reads still rebind the hook`, + ).toBeGreaterThan(starts) + reach.reach(`query-input-change`) + } } finally { resources.defer(() => source.cleanup()) } @@ -412,7 +437,7 @@ async function runHistory(history: History) { } it.each([...histories()])( - `preserves $law through $input, nested=$nested native=$native`, + `preserves $law through $input, nested=$nested native=$native reactive=$reactiveInput`, runHistory, ) From 0b53825d4c19446fa3f346c1575820498804da9c Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 07:44:58 -0700 Subject: [PATCH 16/19] fix(db): enroll new demand in sibling replay Law: ARCHITECTURE law9 / Replay participants and failure requires demand started during sibling replay to hold complete publication. Verdicts stay subscription-scoped under Atomic window publication. Reach: 48 new direction/width/order/outcome/waiting-move histories, with24 late active settlements and12 retry starts. Existing receipt reach44/9/12. RED: all48 public retention failures on unchanged repair ec25 and callback a2e75c3 before each production edit. Earlier masked probes are retained. GREEN: each branch passes46 primary,21 neighboring and4 browser tests, including both80-history campaigns and both final-delivery replays. Mutant: admission removal, latest own error, omitted owed error, final restart omission and final commit omission fail intended public comparisons. Limits: controlled core adapters and Node-host SQLite/coordinator streams; finite one-level include histories. Full pnpm test/builds not run. Reach corrections after first runtime trial are explicitly recorded. Preserve arbitrary browser endings with reset/stop terminal regimes and measure actual final-catchup branches. Remove model-only pseudo-calibration, measure sibling reach from adapter receipts, and correct error-scope prose. Lossless review record: seven fixed-now, two accepted-design, no deferrals. --- .changeset/retain-loading-sibling-replay.md | 2 +- docs/contributing/oracle-coverage.md | 34 ++ .../2026-10-30-release-replay-retention.md | 228 +++++++++++++ .../oracle-reviews/callback-metadata.md | 30 ++ .../tests/multi-tab-electric-oracle.test.ts | 64 +++- packages/db/src/collection/subscription.ts | 160 +++++---- packages/db/src/query/live/ARCHITECTURE.md | 4 +- .../query/live/collection-config-builder.ts | 4 +- ...ad-subset-replay-refinement-oracle.test.ts | 312 ++++++++++++++---- 9 files changed, 693 insertions(+), 145 deletions(-) diff --git a/.changeset/retain-loading-sibling-replay.md b/.changeset/retain-loading-sibling-replay.md index 64368eee0d..f7991e70b2 100644 --- a/.changeset/retain-loading-sibling-replay.md +++ b/.changeset/retain-loading-sibling-replay.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Keep the last complete live-query result when a new include demand fails while a sibling subscription is still replaying after truncate. A successful sibling load cannot publish an incomplete replacement. +Keep the last complete live-query result while a new include demand loads under a sibling subscription's truncate replay. The demand now joins the publication barrier when it starts, so a sibling that settles first cannot publish an incomplete replacement. A failed held demand keeps that result until its own coverage recovers or its demand retires. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index b4944f2ec7..8892d2fddd 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6577,3 +6577,37 @@ scoped health comparison. All existing comparisons, histories, budgets and reach minima remain. The controlled core, one-level include and peer limits remain those of this owner; no provider or framework claim follows. See the [versioned repair record](oracle-reviews/2026-10-30-release-replay-retention.md#subscription-scoped-request-error-checker-repair). + + +### First-demand admission and terminal regimes (2026-11-02) + +The load-subset replay owner adds 48 finite histories crossing sibling/include +settlement order, direction, width, failure/success and a waiting window. A new +parent remains eligible after sibling success. Before admission repair, all 48 +fail the existing public-retention comparison. The repaired driver measures 24 +late active include settlements and 12 retries while a sibling loads, directly +from adapter receipts. The original sibling grammar's receipt reaches are now +44 failures, 9 same-truncate failed-demand unloads during commands and 12 own +recoveries; its earlier model-derived retirement count 33 is historical. All +existing semantic comparisons and required minima stay. The original-admission +mutant fails retention. + +The earliest-error comparison was corrected from source-wide failure order to +subscription-owned verdicts. Authority is the pre-existing maintainer decision +in ARCHITECTURE **Atomic window publication**: a move rejects with *that replay's +own* earliest still-deciding failure. Shared publication does not merge verdicts. +This is an explicit account of the correction, not a claim that the assertion +was unchanged. There is no source-wide race winner promise. The same-subscription +latest-error mutant fails; the actual public owed-error check also rejects an +error omitted after sibling coverage. Moves that resume into a different failed +replay retain the owner's stated limit. A failed move is not promised success +merely because a later retry recovers its demand. + +The multi-tab Electric owner now generates untouched arbitrary, reset-abandoned +and stopped-source endings. Existing final comparisons, 80-run budgets, known +gaps and minima stay; new reach separately measures final stopped-source restart +and lagging-row rewrite. Old grammar reach was 0/46, so only the first was absent. +Final-restart and final-commit omissions fail the real SQLite/coordinator public +comparisons. These are Node-host controlled-stream receiving histories, not real +provider or browser-process claims. The core and browser path limits are unchanged. +See the [nine-item review amendment](oracle-reviews/2026-10-30-release-replay-retention.md#first-demand-admission-and-terminal-grammar-review-2026-11-02). diff --git a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md index 06e78d2e28..c0739316ea 100644 --- a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md +++ b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md @@ -215,3 +215,231 @@ finite calibration, with one worker and dependency verification disabled. The amended primary owner SHA256 is `a2ab454ff5a77aa47bf57cbd590b67a2a5602d3dd6897af051fb1819b9482ce7` on both branches. The earlier table binds the original repair evidence. Its runtime and browser hashes remain unchanged; callback runtime hashes remain bound by its separate port record. + + +## First-demand admission and terminal grammar review (2026-11-02) + +This amendment evaluates all nine findings from the human-supplied review of +`ec25fc4f117c0b20fa0827f33b2a400a6e77a36a`. The original review was unverified; +its examples were hypotheses. This is an author self-review. The raw review, +append-only nine-item ledger, source bindings and complete logs are retained in +`/private/tmp/callback-metadata/release-replay-repair/nine-item-review-20261102`. + +### Law, experiment and inference + +**Law and authority.** ARCHITECTURE **Replay participants and failure**, law 9, +requires new demand started under a sibling's replay to share publication. +**Atomic window publication** separately permits a move to reject once a waited +replay fails. The earliest-error promise is that replay's own earliest still +deciding failure, an explicit maintainer decision already present on release +base `e75f8455a78bc33c43376288218e9912dc2a3f4c`; it imposes no global failure order +across independent subscription verdicts. Laws 16/17 scope request errors to +the failed demand. Publication retention and operation success are different +judgments. + +**Experiment before production.** The existing primary owner gained a provider +settlement selector and a 48-history grammar: direction, width, include/sibling +settlement order, success/failure and presence of a waiting move. An unmarked +sixth parent remains eligible after the sibling succeeds. All 48 histories fail +on unchanged ec25 at the existing public retention assertion: row 6 publishes +while the retained result is empty and include work remains held. The mismatch +can occur before include failure, so changing only `failDemand` cannot close +this ordering. Earlier probes without the persistent parent passed; obsolete +speculative includes retired, or other held loads masked the publication cut. +Those passing probes were not treated as refutations. Both 96-history error +campaigns passed before production changed. The real core receiving path is an +ordered live-query Collection, its join and one-level inline include over real +Collections with controlled adapter promises. No new framework/provider promise +requires a different receiving driver for this admission repair. + +**Repair.** First demand now creates its own replay hold when a sibling replay +still loads, before calling the adapter. Setup itself holds publication through +reentrancy, and the existing participant machinery handles settlement, abort, +retirement and retry. Sibling success can no longer remove this subscription's +hold. The failure fallback remains for the existing failure path. + +**Reach.** The final selector chooses active, un-aborted acquisition receipts; +it drains other work while keeping the selected include held. The 48 histories +reach 24 late active settlements and 12 retry starts while a sibling loads. +The selector and retry counters were corrected after the first runtime trial; +the original 48-history public-retention RED remains the pre-edit gate. Do not +claim the corrected reach counters ran before that trial. The first full run +passed all behavior comparisons but had three reach-only failures: one selector +included retired speculative requests, and two unconstrained campaigns inherited +new obligations owned by the constructive grammar. These were corrected without +removing any pre-existing reach or minimum. + +New-sibling classification, failure and retirement reach no longer read model +barriers. The driver snapshots acquisitions at the actual truncate and uses +adapter starts, failures, aborts and unloads. The original 48-cell grammar now +measures 44 failures, 9 same-truncate failed-demand unloads during commands, and +12 later own recoveries. The historical retirement count 33 was model-derived; +it is not the count of this stricter receipt witness. Final cleanup and a newer +truncate's physical-acquisition renewal supply no retirement reach. Semantic +retention, request-error and recovery comparisons remain unchanged. + +**Calibration.** Disabling admission reproduces the intended retention RED. +Reporting the latest own failure instead of the earliest fails error identity. +Omitting an actually observed public error after successful sibling coverage +fails the ordinary owed-request-error assertion. The removed model-only +`replayScopeViolation` assertion proved no production behavior; its replacement +evidence is this public observation calibration, not a second model invariant. +All temporary subjects were restored in `finally` and their hashes verified. + +### Browser terminal regimes + +The old suffix's fixed 80-history run reaches zero stopped-source restarts at +final catch-up. Its existing final comparisons do exercise a lagging-row rewrite +in 46 histories, so the review's assertion that both regimes are absent is too +broad. The missing stopped-source reach is a grammar failure, not a production +mismatch. A terminal dimension now keeps the original arbitrary final state or +constructs a reset-abandoned commit or a stopped source. The 24-step stop prefix, +8–24-step arbitrary tail, 80 accepted histories per campaign, all comparisons, +existing minima and known-gap handling remain. The constructive endings add +`4 + 2 * tabCount` steps (8 or 10); an untouched ending adds none. + +The real SQLite/coordinator driver measures both final branches. Fixed/random +campaigns pass with final stopped-source restart reach 53/42, lagging-row rewrite +58/52 and reset-abandoned final recovery 22/35. Temporary omission of final +restart fails the public source-head comparison; omission of final commit fails +an ordinary durable/public comparison. This is the separate release CI reach +repair described earlier, not evidence about include admission. Sampling the +terminal dimension is bounded evidence, not a universal guarantee of reach for +every possible future random seed. Required minima continue to fail any campaign +that misses a declared state. + +### Lossless item accounting + +| ID | Technical judgment | Action and durable value | +| --- | --- | --- | +| R1 | Confirmed production and history gap; the narrower failure-only fix missed admission. | Fixed first-demand admission and added settlement order with a persistent parent. Original-admission mutant fails retention. | +| R2 | Rejection despite eventual successful retry is allowed by Atomic window publication. An already failed operation need not become successful. | Accepted design. Keep rejection separate from shared retention; the new admission path participates normally rather than depending on synthetic failed completion. Retry reach is explicit. No success-after-retry promise was added. | +| R3 | The self-check follows directly from the model's guarded deletion loop and does not calibrate production. Existing health comparisons have independent sensitivity. | Removed the redundant model self-check. A wrong actual health observation is rejected at the owed-error comparison. | +| R4 | Cross-subscription earliest-error ordering is not promised. The scope change aligns with an existing maintainer decision. Its reporting needed to be explicit. | Accepted design; coverage map now cites the authority and records the comparison correction. A latest-own-error mutant fails. A claim that no assertion changed would be inaccurate: the old source-wide expectation was corrected, not preserved. | +| R5 | Stopped-tab final recovery was absent; lagging-row final rewriting was reached. The suffix still removed arbitrary final states. | Added terminal regimes and actual branch reach. Kept the separate browser release-CI purpose explicit. Final restart/commit mutants fail. | +| R6 | The claim of receipt-only reach was inaccurate because classification read model barriers. | All newly claimed reach comes from actual adapter receipts and truncate/command boundaries. Model-based historical counts are not relabeled as receipt measurements. | +| R7 | Confirmed duplicate classifier. | One classifier serves both marker adapter paths. | +| R8 | Confirmed incomplete explanatory comment. | Comment now names sibling settlement and retirement. Architecture clarifies admission before adapter reentry. | +| R9 | Confirmed missing suffix length. | Prose states prefix, arbitrary tail and optional 8/10-step ending separately. | + +Totals: seven `fixed-now`, two `accepted-design` (R2/R4); no deferred finding. +R4's reporting correction is complete even though its proposed extra product +obligation is not accepted. Every original claim and suggested repair has a +separate ledger entry. No global source-wide error order or automatic success +of an already failed operation was introduced. + +### Reviewer assessment and limits + +Qualified hire. The reviewer found a real admission defect and useful test +integrity, reach and prose problems. The proposed admission location was sound. +The review explicitly labeled its production claims unverified. It still +confused operation rejection with publication retention, missed an explicit +maintainer decision, and overstated the browser reach loss. Executable probes +and authority checks are needed before treating its downstream claims as facts. + +The claim remains bounded by this owner's finite histories, one ordered source, +one-level include, no peer for retention, controlled core adapter and named +public checkpoints. Existing initial-readiness, abort, retirement, request-error, +error-identity and convergence checks remain. This is not Effect disposal, +arbitrary provider, framework, deep-include or multi-process browser closure. +An evaluated review does not establish whole-PR readiness: final pushed CI and +integration with the actual main branch must be assessed separately. Neither +PR is authorized to merge by this review. + + +### Final validation and source binding + +The browser counts above bind the earlier expanded-grammar run in +`browser-expanded.log`. Final runs use the same grammar, comparisons and +original 80-accepted-history budgets. Their measured final branches are: + +| Branch/run | Stopped-source restart | Lagging-row rewrite | Reset-abandoned final recovery | +| --- | ---: | ---: | ---: | +| Repair fixed | 53 | 58 | 22 | +| Repair random | 48 | 52 | 31 | +| Callback fixed | 53 | 58 | 22 | +| Callback random | 46 | 58 | 31 | + +On each branch, one fresh full primary run passes all 46 tests, the three +neighboring files pass 21 tests, and the browser selection passes both +campaigns plus both existing final-delivery replays (four tests). Focused core +and browser TypeScript, ESLint, formatting, whitespace and `pnpm lint:changed` +pass on both branches. ESLint retains two existing shadow warnings. Whole +repository `pnpm test`, builds and installs did not run. Local Vitest is cached +3.2.4; these runs do not replace final-head CI. + +Callback browser setup initially missed Electric, debug and SQLite resolution. +A subsequent scratch mapping separated the mock from the adapter's SDK module, +so no controlled stream started. Those runs are setup failures, not behavioral +RED. The final mapping resolves both SDK imports to the same installed module +and runs the callback branch's actual sources. No repository dependency file +or installation changed. + +The repair is based on `ec25fc4f117c0b20fa0827f33b2a400a6e77a36a`; the port is +based on `a2e75c30402deb815abd01783ea816a2edfe3ea8`, which already integrates +main `85392bd1d349aa3bb77c9c11ffba671f0b2d2a84`. Both final trees share these +SHA256 bindings: + +| Source | SHA256 | +| --- | --- | +| `packages/db/src/collection/subscription.ts` | `5cd5a8759dc38bf70c88ed756b6886e474afef5ad02f4419bd0f5d1ab6b39dc0` | +| `packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts` | `d3c4edf87d4f046d888887ab4dea6e16ed40571a8e4334850427573d92c51271` | +| `packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts` | `3184cd014be5ab6222350525bbd3d09a78b46e49c5e8b17fdde6ffbc417f26c5` | + +The containing commit versions this evidence; final commit IDs and all changed +path hashes are additionally bound in the scratch `final-binding.json` and the +published PR descriptions. The builder differs between branches because the +callback branch retains its plural joined-dependency gate. This review changes +only its explanatory comment. Compiler and Solid source/test paths are +unchanged from each branch's starting head. Earlier runtime fingerprints and +size figures remain historical, because admission intentionally changes runtime. + +### Oracle guide audit for this amendment + +This is author self-review, not an independent external review. + +- ORC-001/002/003: the architecture and existing maintainer decisions supply + authority. The independent source-row/barrier model and server log remain + separate from real drivers. Local prose explains the new grammar and cuts. +- ORC-004/013: direction, widths 1–3, both settlement orders, success/failure and + waiting/no-waiting moves reconstruct the new core witness. Removing the + persistent parent masks it through retirement; removing order variation + misses sibling-first publication. Browser arbitrary/reset/stop endings keep + the prior arbitrary domain and distinguish final recovery branches. Aborting + an acquisition excludes it from the active demand premise. Closing a tab for + good remains outside the existing browser convergence claim. +- ORC-005/006: core compares actual public rows and request errors after commands, + quiet loading and authoritative recovery. SQLite/coordinator compares durable + and public rows/status at existing cuts. Admission, own-error, owed-error, + final-restart and final-commit controls reject at intended assertions, not + setup or timeout. All temporary subjects were restored. +- ORC-007: fixed and random campaigns retain the same generators, comparisons + and budgets. The existing direct seed/path interface and two final-delivery + replays remain. The new 48-cell core grammar is bounded enumeration, so it + does not need a second random copy. Existing related-issue seed/path results + remain bounded historical evidence. +- ORC-008/009: removing model-only calibration state cannot change a promised + result. Acquisition receipt state belongs to the driver and measures reach, + not expected publication. Publication stays source-shared while verdicts stay + subscription-owned. These distinctions use the glossary's demand/acquisition + boundary and the documented barrier abstraction. +- ORC-010: finite enumeration preserves every mismatch in an AggregateError, + with its first failure as cause. Existing cleanup still releases real resources. + Setup and reach failures remain separate from behavioral mismatches. +- ORC-011: the concrete shared-fault hypothesis was model-derived reach. The + repair measures actual adapter receipts independently of model barriers. No + additional shared semantic fault requiring a second model was identified. +- ORC-012/015: this versioned record reports limits, the original public RED, + final GREEN, hostile controls, both branches' hashes and declared reach. New + constructive reach is not imposed on unrelated arbitrary campaigns. +- ORC-014: the claims end at controlled core adapters and Node-host real SQLite/ + coordinator code. Controlled Electric streams and peer-reset messages supply + premises. No real-provider or browser-process handoff claim follows. +- ORC-016: new finite histories use the existing shared model/driver/comparison. + Final-delivery replays carry inputs without separate expected values. No unit + assertion or concrete type assertion was added to an oracle. + +No raw review item is deferred. The remaining limitations are the owners' +explicit controlled and finite boundaries, not a known reachable violation of +the repaired admission law. Final pushed CI and repair integration with actual +main remain separate readiness obligations. No merge authorization follows. diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 59b4a06f8e..6b3c3dd7cf 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -1069,3 +1069,33 @@ CI for integration head `fa5df11a85342ad6fee373aedb3b7391e7b25630` exposed eight **Replays and evidence:** scratch logs and task-local configs are under `/private/tmp/callback-metadata/solid-integration-20261102`. `pre-edit-binding.json` predates the production edit. `extended-parent-red.log`, `green.log`, `solid-suite-green.log`, `original-mutant.log`, and `over-untracked-mutant.log` retain the actual results. The focused runner commands use `pnpm_config_verify_deps_before_run=false pnpm exec vitest run --config ` with at most two workers. The RED and restored-hook campaigns select `lifetime`; the over-untracked campaign selects `reactive=true`. `config.mjs` runs the primary owner, `full.config.mjs` runs the eight runtime files, and the two mutant configs substitute only the hook under test. `tsconfig.json` supplies cached dependency paths for focused TypeScript and ESLint checks. These mappings are scratch artifacts and do not change package configuration. **Limits:** finite histories on the actual Solid host at its settled cut. The change does not specify native-source-only value liveness, arbitrary query callbacks that intentionally read live row data as parameters, every asynchronous source-start schedule, other framework adapters, or real browser rendering. Runtime suite counts exclude type-only test registration. Focused TypeScript passes separately. Whole-repository `pnpm test` and builds were not run. This is author self-review. External CodeRabbit coverage remains paused on an older commit and supplies no current-head review. + + +## Shared replay first-demand admission port (2026-11-02) + +The nine-item review of PR 2318 also exposes the admission gap in this branch. +Before production changed from `a2e75c30402deb815abd01783ea816a2edfe3ea8`, the +extended primary owner's 48 settlement histories fail the public retention +comparison. Row 6 publishes while the model requires the empty retained result. +The source and new oracle are bound separately in the review scratch record. + +The shared runtime/oracle/browser repair and its limits are documented in the +[release review amendment](2026-10-30-release-replay-retention.md#first-demand-admission-and-terminal-grammar-review-2026-11-02). +This port preserves the callback compiler, joined-source gates and Solid input +preparation fix. It changes no callback metadata law. The latest external +CodeRabbit review remains paused at historical commit 8464; it supplies no +review of this semantic port. Tests and current CI are recorded separately. + + +The final port passes 46 primary tests, 21 neighboring tests and four browser +tests (both 80-history campaigns plus both final-delivery replays). Its final +browser restart/rewrite/reset-recovery reaches are 53/58/22 fixed and 46/58/31 +random. Focused core/browser TypeScript, ESLint (zero errors, two existing +warnings), formatting, whitespace and changed-file lint pass. Earlier callback +browser attempts were module-resolution/mock setup failures and supply no +behavioral RED. The final scratch mapping uses one installed Electric module +for both the controlled stream and adapter. No dependencies were installed. +The shared runtime, primary and browser hashes match the release amendment's +final binding table. Compiler and Solid source/test paths remain exactly those +of a2e75c3. The containing commit binds this port; full commit IDs and changed +path hashes are retained in scratch final-binding.json and the PR body. diff --git a/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts index 341715569a..1dc3261503 100644 --- a/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts @@ -973,7 +973,9 @@ describe(`multi-tab Electric over one persisted database`, () => { // Every stop kind occurs in a generated order, with a distinct running // peer. Recovery and catch-up separate the blocks. Four membership toggles // of an initially absent key restore the original rows before the tail. - // This adds 24 steps to the unchanged 8–24-step arbitrary tail. Removing + // The prefix adds 24 steps to the unchanged 8–24-step arbitrary tail. An + // ending dimension either leaves that final state untouched or adds a + // constructive reset/stop suffix of 4 + 2*tabCount steps (8 or 10). Removing // the open delivery, stop, or peer commit loses the law's premise; recovering // before the peer commit would test a different boundary. Existing pinned // histories still use the same driver without this prefix. @@ -983,6 +985,11 @@ describe(`multi-tab Electric over one persisted database`, () => { source: fc.nat(2), key: fc.constantFrom(3, 4), recovery: fc.nat(1), + ending: fc.constantFrom( + `arbitrary` as const, + `reset` as const, + `stop` as const, + ), stopOrder: fc.shuffledSubarray([0, 1, 2, 3], { minLength: 4, maxLength: 4, @@ -995,6 +1002,7 @@ describe(`multi-tab Electric over one persisted database`, () => { source, key, recovery, + ending, stopOrder, steps, }): GeneratedHistory => { @@ -1012,26 +1020,32 @@ describe(`multi-tab Electric over one persisted database`, () => { { kind: `catch up`, tab, pick: 0 }, ]), ...steps, - // Final catch-up must encounter an abandoned reservation in every - // generated history. Start a fresh handle, recover every tab, and - // deliver one new change without committing it. Reset then abandons - // that reservation. No later history step commits or recovers it; - // the existing final-delivery comparison must observe its failure - // before the existing recovery establishes convergence (Laws 2/5). + // Preserve arbitrary final states as well as constructing the rare + // final-reset and stopped-tab boundaries. The constructive endings + // start a fresh handle, recover every tab, then deliver one change + // without committing it. Reset abandons that reservation, so final + // delivery judges its failure before recovery (Laws 2/5). Stop + // instead leaves final catch-up to restart the stopped source. // The source tab, key and two/three-tab host vary with this grammar. // Removing advance or reset loses that premise; catching up here // would move the failure out of the final observation cut. - { kind: `deploy` }, - ...Array.from( - { length: tabCount }, - (_, index): Array => [ - { kind: `recover`, tab: index, pick: recovery }, - { kind: `catch up`, tab: index, pick: 0 }, - ], - ).flat(), - { kind: `server change`, key, pick: 0 }, - { kind: `advance`, tab, pick: 0 }, - { kind: `peer reset` }, + ...(ending === `arbitrary` + ? [] + : [ + { kind: `deploy` as const }, + ...Array.from( + { length: tabCount }, + (_, index): Array => [ + { kind: `recover`, tab: index, pick: recovery }, + { kind: `catch up`, tab: index, pick: 0 }, + ], + ).flat(), + { kind: `server change` as const, key, pick: 0 }, + { kind: `advance` as const, tab, pick: 0 }, + ending === `reset` + ? { kind: `peer reset` as const } + : { kind: `stop` as const, tab, pick: stopOrder[0]! }, + ]), ], } }, @@ -1526,9 +1540,19 @@ describe(`multi-tab Electric over one persisted database`, () => { models[index]!.stopped !== undefined || models[index]!.failedByReset ) { + generatedReach.reach(`stopped source restarted during final catch-up`) await recover(index, `restart`) } + const laggingBeforeFinal = + rowsOf(durable).some(({ id, label }) => log.rows.get(id) !== label) || + rowsOf(log.rows).some(({ id, label }) => durable.get(id) !== label) await deliver(index, `all`, true) + if ( + laggingBeforeFinal && + !models[index]!.failedByReset && + JSON.stringify(rowsOf(durable)) === JSON.stringify(rowsOf(log.rows)) + ) + generatedReach.reach(`lagging commit rewritten during final catch-up`) // A reset-abandoned transaction fails only when delivery tries to // commit it. Judge Law 5 at that cut before the restart establishes // Law 2's successful catch-up premise. @@ -1716,6 +1740,8 @@ describe(`multi-tab Electric over one persisted database`, () => { ) type GeneratedReach = + | `stopped source restarted during final catch-up` + | `lagging commit rewritten during final catch-up` | `must-refetch delivered` | `peer commit during an open reserved transaction` | `source stopped with an open transaction` @@ -1738,6 +1764,8 @@ describe(`multi-tab Electric over one persisted database`, () => { const createGeneratedReach = () => createOracleReach(`browser.multi-tab-reserved-history`, [ + `stopped source restarted during final catch-up`, + `lagging commit rewritten during final catch-up`, `must-refetch delivered`, `peer commit during an open reserved transaction`, `source stopped with an open transaction`, diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 949e0eb4d4..552c2f96c5 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -1185,83 +1185,109 @@ export class CollectionSubscription result: LoadSubsetRequestResult started: boolean } { - const demand: SubsetDemand = { - requestOptions, - acquisition: { - options: requestOptions, - syncRunGeneration: this.collection._sync.getSyncRunGeneration(), - }, - acquisitionState: `starting`, - } - // A demand that starts during an open replay joins it, parked or not (law - // 9: a new demand joins the open replay). - const replayState = this.truncateReplayState - const replayAttempt = replayState?.currentAttempt + // Law 9: a first demand joins the shared publication barrier when it + // starts, even if this subscription held no demand at truncate. Waiting + // until failure loses the hold when the sibling settles first. + let joinedReplay: TruncateReplayState | undefined if ( - this.collection.status === `cleaned-up` || - // A deferring subscription starts no acquisition attempt until it resumes. - (this.defersAcquisition && - this.collection.config.syncMode === `on-demand`) || - // Ready/error callbacks can run before sync returns its loader. Idle - // deferred starts still acquire through the sync manager's queue. - (this.collection.config.syncMode === `on-demand` && - (isFailedStatus(this.collection.status) || - (this.collection.status !== `idle` && - this.collection._sync.syncLoadSubsetFn === null))) + !this.truncateReplayState && + this.options.truncateReplayPublication?.siblingReplays() ) { - demand.acquisitionState = `detached` + joinedReplay = this.createTruncateReplayState( + this.collection._sync.getSyncRunGeneration(), + () => new Map(), + ) + this.truncateReplayState = joinedReplay + this.truncateReplacementPending = true + } + const joinedAttempt = joinedReplay?.currentAttempt + try { + const demand: SubsetDemand = { + requestOptions, + acquisition: { + options: requestOptions, + syncRunGeneration: this.collection._sync.getSyncRunGeneration(), + }, + acquisitionState: `starting`, + } + // A demand that starts during an open replay joins it, parked or not (law + // 9: a new demand joins the open replay). + const replayState = this.truncateReplayState + const replayAttempt = replayState?.currentAttempt + if ( + this.collection.status === `cleaned-up` || + // A deferring subscription starts no acquisition attempt until it resumes. + (this.defersAcquisition && + this.collection.config.syncMode === `on-demand`) || + // Ready/error callbacks can run before sync returns its loader. Idle + // deferred starts still acquire through the sync manager's queue. + (this.collection.config.syncMode === `on-demand` && + (isFailedStatus(this.collection.status) || + (this.collection.status !== `idle` && + this.collection._sync.syncLoadSubsetFn === null))) + ) { + demand.acquisitionState = `detached` + this.subsetDemands.push(demand) + const initialResult = createDeferred() + demand.initialResult = initialResult + const abort = () => + initialResult.reject(new LoadSubsetOperationAbortedError()) + requestOptions.signal?.addEventListener(`abort`, abort, { once: true }) + const finish = () => { + requestOptions.signal?.removeEventListener(`abort`, abort) + demand.initialResult = undefined + } + void initialResult.promise.then(finish, finish) + if (replayState && replayAttempt) + demand.parkedParticipant = this.trackTruncateReplayParticipant( + replayState, + replayAttempt, + demand, + initialResult.promise, + requestOptions.signal, + ) + return { demand, result: initialResult.promise, started: false } + } + const acquisition = this.createSubsetAcquisitionRecord(demand) + demand.acquisition = acquisition + const syncRunGeneration = this.collection._sync.getSyncRunGeneration() + // Reentrant release must see the exact acquisition before adapter work + // starts. A load that throws is a rejection: it keeps its owner. this.subsetDemands.push(demand) - const initialResult = createDeferred() - demand.initialResult = initialResult - const abort = () => - initialResult.reject(new LoadSubsetOperationAbortedError()) - requestOptions.signal?.addEventListener(`abort`, abort, { once: true }) - const finish = () => { - requestOptions.signal?.removeEventListener(`abort`, abort) - demand.initialResult = undefined + const result = this.collection._sync.loadSubset(acquisition.options) + + if (!this.isSyncRunGenerationCurrent(syncRunGeneration)) { + const demandIndex = this.subsetDemands.indexOf(demand) + if (demandIndex !== -1) this.subsetDemands.splice(demandIndex, 1) + cancelAcquisition(acquisition) + return { demand, result, started: true } } - void initialResult.promise.then(finish, finish) - if (replayState && replayAttempt) - demand.parkedParticipant = this.trackTruncateReplayParticipant( + + demand.acquisitionState = `active` + if (!this.subsetDemands.includes(demand)) { + this.releaseAcquisition(acquisition) + return { demand, result, started: true } + } + + if (replayState && replayAttempt) { + this.trackTruncateReplayParticipant( replayState, replayAttempt, demand, - initialResult.promise, - requestOptions.signal, + result, + acquisition.options.signal, ) - return { demand, result: initialResult.promise, started: false } - } - const acquisition = this.createSubsetAcquisitionRecord(demand) - demand.acquisition = acquisition - const syncRunGeneration = this.collection._sync.getSyncRunGeneration() - // Reentrant release must see the exact acquisition before adapter work - // starts. A load that throws is a rejection: it keeps its owner. - this.subsetDemands.push(demand) - const result = this.collection._sync.loadSubset(acquisition.options) - - if (!this.isSyncRunGenerationCurrent(syncRunGeneration)) { - const demandIndex = this.subsetDemands.indexOf(demand) - if (demandIndex !== -1) this.subsetDemands.splice(demandIndex, 1) - cancelAcquisition(acquisition) - return { demand, result, started: true } - } - - demand.acquisitionState = `active` - if (!this.subsetDemands.includes(demand)) { - this.releaseAcquisition(acquisition) + } return { demand, result, started: true } + } finally { + // Adapter and status callbacks can reenter before returning a promise. + // Setup holds the replacement until that tentative work is registered. + if (joinedReplay && joinedAttempt) { + joinedReplay.pendingSetups-- + joinedAttempt.setupComplete = true + this.checkTruncateReplayComplete(joinedReplay) + } } - - if (replayState && replayAttempt) { - this.trackTruncateReplayParticipant( - replayState, - replayAttempt, - demand, - result, - acquisition.options.signal, - ) - } - return { demand, result, started: true } } /** Re-check ownership after adapter and event callbacks that may reenter. */ diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 158d40dcf6..5ca54cc646 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1565,7 +1565,9 @@ still has replay work, so demand that any of them starts meanwhile joins its own open replay. While a replay of any of them still holds publication, a lazy demand of any of them that fails after initial-query readiness fails its own subscription's replacement too. This includes a subscription that held no -demand at truncate. Settling a sibling's replay or retiring its failed demand +demand at truncate. Such a subscription joins the barrier when its first +acquisition starts, before adapter code can reenter; its current work still +holds publication after the sibling settles. Settling a sibling's replay or retiring its failed demand cannot publish the window without this demand's rows. It holds until every demand load that joins it loads, a later replay of the source succeeds, or the subscription's last demand retires. The query retries the failed demand diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 26acc639d8..37cf9f0fb9 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -598,8 +598,8 @@ export class CollectionConfigBuilder< // it, and ordered work must not wait for a load that already ended. // Output held for this demand publishes now, unless a replay of this // source still holds publication: the failure then holds this - // subscription's publication too, so retiring the other replay's - // failed demand cannot publish without this demand's rows. + // subscription's publication too. Neither sibling settlement nor + // retirement of its failed demand may publish without these rows. demand.settled = true if ( this.collectionSources.some( diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts index 7ad2fdfd13..00a7aafafc 100644 --- a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts @@ -179,7 +179,14 @@ type OrderedCommand = // while the next stays pending. `newest` settles the most recently held // load instead of the oldest, so a later load can settle before an earlier // one, as independent provider requests may. - | { type: `settle`; ok: boolean; keepHolding?: boolean; newest?: boolean } + | { + type: `settle` + ok: boolean + keepHolding?: boolean + newest?: boolean + // Settle a new sibling demand, or drain other adapter work while it stays held. + target?: `new-sibling` | `other` + } | { type: `truncate`; armAfterStart: boolean } | { type: `unloadThrows` } | { type: `window`; offset: 0 | 1; limit: 1 | 2 | 3 } @@ -322,6 +329,8 @@ type OrderedReach = { siblingSuccessfulCoverage: number // Owed request-error comparisons after another subscription asked for its keys. siblingRequestErrorChecks: number + lateSiblingSettlements: number + retryWhileSiblingLoads: number newSiblingReplayFailures: number newSiblingReplayRetirements: number newSiblingReplayRecoveries: number @@ -477,10 +486,12 @@ async function checkOrderedReplay( // This subscription held no acquisition at truncate, then started demand // during its sibling replay. Measured only from adapter receipts. newSiblingReplay?: boolean + markerTruncate?: number keys?: Set ordered?: boolean } const loads: Array = [] + const receiptOptions = new Map() // Retention model. Each source with an open or failed replay has a // barrier. `pending` holds the loads that still hold its publication: the // loads started while it is up, and work an older replay started. `own` @@ -499,8 +510,6 @@ async function checkOrderedReplay( ambiguous: boolean pending: Map own: Set - // Acquisition receipts held at truncate, used only for declared reach. - subscriptionsAtTruncate: Set } const barriers = new Map() // Every barrier this history raised, in raise order. @@ -530,18 +539,6 @@ async function checkOrderedReplay( ambiguous: [...barriers.values()].some(({ ambiguous }) => ambiguous), pending: new Map(barriers.get(source)?.pending), own: new Set(), - subscriptionsAtTruncate: new Set( - [...acquired].flatMap(([options, count]) => - count > 0 && - loads.some( - (load) => - load.source === source && - load.subscription === options.subscription, - ) - ? [options.subscription] - : [], - ), - ), } barriers.set(source, barrier) raisedBarriers.push(barrier) @@ -568,7 +565,6 @@ async function checkOrderedReplay( barrier.own.add(load) return true } - let replayScopeViolation: string | undefined const settleParticipant = ( source: BarrierSource, load: Load, @@ -590,13 +586,6 @@ async function checkOrderedReplay( barrier.own.has(load) && !options.signal?.aborted ) { - if ( - load.newSiblingReplay && - [...barrier.pending.values()].some( - (pending) => pending.subscription !== options.subscription, - ) - ) - reach.newSiblingReplayFailures++ barrier.failures.set(load, options) barrier.history.push(load) } @@ -605,12 +594,9 @@ async function checkOrderedReplay( // failure no longer fails the replay (ARCHITECTURE.md law 17: a failure // counts for a key no later held load loaded). if (ok && barrier?.pending.has(load) && !options.signal?.aborted) { - // Calibration of the replay account: a successful sibling can supply the - // same source keys without deciding another subscription's verdict. - // Preserve those failure identities independently of the mutation below. - const siblingFailures = [...barrier.failures.keys()].filter( - (failed) => failed.subscription !== load.subscription, - ) + // Sibling success cannot decide another subscription's failure. + // The public request-error comparison below enforces that distinction; + // model self-consistency is not production calibration evidence. for (const failed of barrier.failures.keys()) { if ( failed.subscription !== load.subscription && @@ -630,8 +616,6 @@ async function checkOrderedReplay( barrier.failures.delete(failed) } } - if (siblingFailures.some((failed) => !barrier.failures.has(failed))) - replayScopeViolation ??= `successful sibling removed another subscription's replay failure` } barrier?.pending.delete(load) // A replay completes when its last participant settles, so demand that @@ -649,8 +633,6 @@ async function checkOrderedReplay( for (const owned of [barrier.pending, barrier.failures]) { for (const [load, held] of owned) { if (held === options) { - if (owned === barrier.failures && load.newSiblingReplay) - reach.newSiblingReplayRetirements++ owned.delete(load) } } @@ -667,6 +649,46 @@ async function checkOrderedReplay( const oneSubscription = (_source: BarrierSource) => !scenario.peer const acquired = new Map() const held: Array<(ok: boolean) => void> = [] + const heldReceipts = new Map<(ok: boolean) => void, Load>() + const holdLoad = (load: Load) => + new Promise((resolve) => { + held.push(resolve) + heldReceipts.set(resolve, load) + }) + // Driver receipts at the actual truncate, independent of model membership. + // These select legal settlement orders and measure reach only. + let markerSubscriptionsAtTruncate = new Set< + LoadSubsetOptions[`subscription`] + >() + const hasPendingReplaySibling = ( + subscription: LoadSubsetOptions[`subscription`], + ) => + loads.some( + (load) => + load.source === `markers` && + load.outcome === `pending` && + load.subscription !== subscription && + markerSubscriptionsAtTruncate.has(load.subscription) && + !receiptOptions.get(load)?.signal?.aborted && + (acquired.get(receiptOptions.get(load)!) ?? 0) > 0, + ) + const isNewSiblingReplayLoad = (options: LoadSubsetOptions) => + markerTruncates > 0 && + !markerSubscriptionsAtTruncate.has(options.subscription) && + ((markerCommitting && + [...markerSubscriptionsAtTruncate].some( + (subscription) => subscription !== options.subscription, + )) || + hasPendingReplaySibling(options.subscription)) + const noteNewSiblingFailure = (load: Load, options: LoadSubsetOptions) => { + if ( + load.newSiblingReplay && + !options.signal?.aborted && + hasPendingReplaySibling(options.subscription) + ) + reach.newSiblingReplayFailures++ + } + let observingHistory = false let holding = false let truncates = 0 let throwNextUnload = false @@ -974,6 +996,8 @@ async function checkOrderedReplay( siblingReplayFailureComparisons: 0, siblingSuccessfulCoverage: 0, siblingRequestErrorChecks: 0, + lateSiblingSettlements: 0, + retryWhileSiblingLoads: 0, newSiblingReplayFailures: 0, newSiblingReplayRetirements: 0, newSiblingReplayRecoveries: 0, @@ -1035,14 +1059,23 @@ async function checkOrderedReplay( source: `markers`, keys: requestedKeys(`markers`, options), } - load.newSiblingReplay = - barriers.get(`markers`) !== undefined && - !barriers - .get(`markers`)! - .subscriptionsAtTruncate.has(options.subscription) && - [...(barriers.get(`markers`)?.pending.values() ?? [])].some( - (pending) => pending.subscription !== options.subscription, + load.markerTruncate = markerTruncates + load.newSiblingReplay = isNewSiblingReplayLoad(options) + receiptOptions.set(load, options) + // This retry premise is measured when the real adapter receives + // another load of a failed demand while its sibling still loads. + if ( + load.newSiblingReplay && + loads.some( + (earlier) => + earlier.subscription === load.subscription && + earlier.outcome === `failed` && + [...(earlier.keys ?? [])].every((key) => + load.keys?.has(key), + ), ) + ) + reach.retryWhileSiblingLoads++ loads.push(load) reach.markerLoads++ // The throw happens while the subscription registers the @@ -1064,6 +1097,7 @@ async function checkOrderedReplay( const recovery = belongsToRecovery(options, markerTruncates, markerCommitting) || (participant && oneSubscription(`markers`)) + noteNewSiblingFailure(load, options) settleParticipant(`markers`, load, options, false) // A load that threw acquired nothing, so the adapter never sees // its demand retire, and any graph run can retire it. The @@ -1092,14 +1126,23 @@ async function checkOrderedReplay( source: `markers`, keys: requestedKeys(`markers`, options), } - load.newSiblingReplay = - barriers.get(`markers`) !== undefined && - !barriers - .get(`markers`)! - .subscriptionsAtTruncate.has(options.subscription) && - [...(barriers.get(`markers`)?.pending.values() ?? [])].some( - (pending) => pending.subscription !== options.subscription, + load.markerTruncate = markerTruncates + load.newSiblingReplay = isNewSiblingReplayLoad(options) + receiptOptions.set(load, options) + // This retry premise is measured when the real adapter receives + // another load of a failed demand while its sibling still loads. + if ( + load.newSiblingReplay && + loads.some( + (earlier) => + earlier.subscription === load.subscription && + earlier.outcome === `failed` && + [...(earlier.keys ?? [])].every((key) => + load.keys?.has(key), + ), ) + ) + reach.retryWhileSiblingLoads++ loads.push(load) reach.markerLoads++ const participant = joinBarrier(`markers`, load, options) @@ -1119,14 +1162,13 @@ async function checkOrderedReplay( const newDemandDuringReplay = !reload && markerCommitting && isDemand(options) if (holding) { - const ok = await new Promise((resolve) => - held.push(resolve), - ) + const ok = await holdLoad(load) if (!ok) { // The source's replay had already failed and still holds // publication when this load fails. const barrierFailed = (barriers.get(`markers`)?.failures.size ?? 0) > 0 + noteNewSiblingFailure(load, options) const replayLoading = settleParticipant( `markers`, load, @@ -1224,6 +1266,20 @@ async function checkOrderedReplay( })() }, unloadSubset: (options) => { + // During commands, same-truncate unload retires a failed demand; + // final cleanup and a newer replay's acquisition renewal do not + // supply this reach witness. + if ( + observingHistory && + loads.some( + (load) => + receiptOptions.get(load) === options && + load.newSiblingReplay && + load.outcome === `failed` && + load.markerTruncate === markerTruncates, + ) + ) + reach.newSiblingReplayRetirements++ unloaded.add(options) retireParticipants(`markers`, options) const count = acquired.get(options) ?? 0 @@ -1262,9 +1318,7 @@ async function checkOrderedReplay( const participant = joinBarrier(`rows`, load, options) belongsToRecovery(options, truncates, truncating) if (holding) { - const ok = await new Promise((resolve) => - held.push(resolve), - ) + const ok = await holdLoad(load) if (!ok) { const replayLoading = settleParticipant( `rows`, @@ -1600,6 +1654,7 @@ async function checkOrderedReplay( await flushPromises() const next = held.shift() if (!next) return + heldReceipts.delete(next) next(true) } throw new Error(`held loads did not drain`) @@ -1624,6 +1679,19 @@ async function checkOrderedReplay( // An eager marker source rewrites its rows; an on-demand one replays the // demand it retains. const truncateMarkers = async () => { + markerSubscriptionsAtTruncate = new Set( + [...acquired].flatMap(([options, count]) => + count > 0 && + !options.signal?.aborted && + loads.some( + (load) => + load.source === `markers` && + load.subscription === options.subscription, + ) + ? [options.subscription] + : [], + ), + ) installedMarkers.clear() raiseBarrier(`markers`) markerSync.begin() @@ -1652,6 +1720,7 @@ async function checkOrderedReplay( try { if (!scenario.historyDuringPreload) await preload let sawRequestError = false + observingHistory = true for (const command of history) { try { switch (command.type) { @@ -1684,7 +1753,40 @@ async function checkOrderedReplay( break case `settle`: { holding = command.keepHolding === true && holding - ;(command.newest === true ? held.pop() : held.shift())?.(command.ok) + const candidates = held.filter((resolve) => { + const receipt = heldReceipts.get(resolve)! + const activeNewSibling = + receipt.source === `markers` && + !markerSubscriptionsAtTruncate.has(receipt.subscription) && + !receiptOptions.get(receipt)?.signal?.aborted + return ( + command.target === undefined || + (command.target === `new-sibling` + ? activeNewSibling + : !activeNewSibling) + ) + }) + const selected = command.newest ? candidates.at(-1) : candidates[0] + if (selected) { + held.splice(held.indexOf(selected), 1) + const receipt = heldReceipts.get(selected)! + if ( + receipt.source === `markers` && + !markerSubscriptionsAtTruncate.has(receipt.subscription) && + !loads.some( + (other) => + other !== receipt && + other.source === `markers` && + other.subscription !== receipt.subscription && + other.outcome === `pending` && + !receiptOptions.get(other)?.signal?.aborted, + ) && + !receiptOptions.get(receipt)?.signal?.aborted + ) + reach.lateSiblingSettlements++ + heldReceipts.delete(selected) + selected(command.ok) + } break } case `truncate`: @@ -1821,6 +1923,7 @@ async function checkOrderedReplay( sawRequestError = true } if (sawRequestError) reach.requestErrors++ + observingHistory = false holding = false throwNextUnload = false // The drain is the checkpoint, where recovery loads succeed, so an @@ -1841,8 +1944,6 @@ async function checkOrderedReplay( if (moveOrderViolation) { throw new Error(`newest move wins: ${moveOrderViolation}`) } - if (replayScopeViolation) - throw new Error(`replay account scope: ${replayScopeViolation}`) if (moveErrorViolation) { throw new Error(`replay error identity: ${moveErrorViolation}`) } @@ -1899,6 +2000,7 @@ async function checkOrderedReplay( } } } finally { + observingHistory = false holding = false for (const resolve of held.splice(0)) resolve(true) for (const subscription of subscriptions) subscription.unsubscribe() @@ -2644,6 +2746,99 @@ describe(`Ordered windows across truncate replays`, () => { declared.assertReached() }, 120_000) + // Law 9 keeps late include demand private after its sibling settles. + // A waiting move distinguishes publication retention from replay verdicts: + // Atomic window publication allows rejection despite a later retry. + it(`keeps publication across sibling and include settlement orders`, async () => { + const states = [`lateSiblingSettlements`, `retryWhileSiblingLoads`] as const + const declared = createOracleReach( + `replay.sibling-settlement-order`, + states, + ) + const failures: Array = [] + for (const direction of [`asc`, `desc`] as const) { + for (const limit of [1, 2, 3] as const) { + for (const includeFirst of [false, true]) { + for (const ok of [false, true]) { + for (const waitingMove of [false, true]) { + declared.beginHistory() + try { + const receipt = await checkOrderedReplay( + [ + { type: `hold` }, + { type: `truncateMarkers` }, + { type: `move`, id: 6, rank: direction === `asc` ? 60 : 1 }, + { + type: `settle`, + ok: true, + keepHolding: true, + target: `other`, + }, + ...(waitingMove + ? [{ type: `window` as const, offset: 0 as const, limit }] + : []), + ...(includeFirst + ? [ + { + type: `settle` as const, + ok, + keepHolding: true, + target: `new-sibling` as const, + }, + ] + : []), + // Twelve settlements exceed the other receipts in this + // six-row domain; selected new demand stays held throughout. + ...Array.from({ length: 12 }, () => ({ + type: `settle` as const, + ok: true, + keepHolding: true, + target: `other` as const, + })), + ...(!includeFirst + ? [ + { + type: `settle` as const, + ok, + keepHolding: true, + target: `new-sibling` as const, + }, + ] + : []), + ], + { + ...ordinaryOrderedScenario, + direction, + offset: 1, + limit, + joined: `none`, + includes: true, + markerMode: `on-demand`, + }, + ) + for (const state of states) + if (receipt[state] > 0) declared.reach(state) + } catch (error) { + failures.push(error) + } + } + } + } + } + } + console.info( + `replay.sibling-settlement-order`, + Object.fromEntries(declared.counts()), + ) + if (failures.length) + throw new AggregateError( + failures, + `${failures.length} settlement histories violated shared publication`, + { cause: failures[0] }, + ) + declared.assertReached() + }, 120_000) + // This finite history grammar varies which source replays, the order of // two load failures, join form, direction and window width. Join plus include // gives sibling subscriptions; the root replay gives same-subscription @@ -2793,6 +2988,8 @@ describe(`Ordered windows across truncate replays`, () => { siblingReplayFailureComparisons: 0, siblingSuccessfulCoverage: 0, siblingRequestErrorChecks: 0, + lateSiblingSettlements: 0, + retryWhileSiblingLoads: 0, newSiblingReplayFailures: 0, newSiblingReplayRetirements: 0, newSiblingReplayRecoveries: 0, @@ -2838,6 +3035,9 @@ describe(`Ordered windows across truncate replays`, () => { `siblingReplayFailureComparisons`, `siblingSuccessfulCoverage`, `siblingRequestErrorChecks`, + // The constructive settlement-order grammar declares these premises. + `lateSiblingSettlements`, + `retryWhileSiblingLoads`, // The constructive new-sibling grammar declares these rare premises. `newSiblingReplayFailures`, `newSiblingReplayRetirements`, From 72990f869ad5cef976d182bbae2aa967ceea7f71 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 08:21:13 -0700 Subject: [PATCH 17/19] test: port final overwrite and deferred replay coverage --- docs/contributing/oracle-coverage.md | 25 +++ .../2026-10-30-release-replay-retention.md | 164 +++++++++++++++ .../oracle-reviews/callback-metadata.md | 22 ++ .../tests/multi-tab-electric-oracle.test.ts | 103 +++++++-- ...-query-deferred-acquisition-oracle.test.ts | 195 +++++++++++++++++- 5 files changed, 485 insertions(+), 24 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 8892d2fddd..ca90d9f708 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -6611,3 +6611,28 @@ Final-restart and final-commit omissions fail the real SQLite/coordinator public comparisons. These are Node-host controlled-stream receiving histories, not real provider or browser-process claims. The core and browser path limits are unchanged. See the [nine-item review amendment](oracle-reviews/2026-10-30-release-replay-retention.md#first-demand-admission-and-terminal-grammar-review-2026-11-02). + + +### Final-catch-up provenance and deferred sibling resumption (2026-11-02) + +The multi-tab Electric owner now separates durable/server divergence from an +actual older same-key commit overwriting a newer peer commit. The old fixed +campaign measured 58 divergence repairs, of which 57 lacked that overwrite +premise and one had it (81 initiated/80 accepted histories). Earlier final +"lagging rewrite" counts are historical divergence evidence. A fourth terminal +regime constructs the actual overwrite. Original arbitrary/reset/stop regimes, +24-step prefix, 8–24-step arbitrary tail, models, comparisons, known gaps, +80-run budgets and minima remain. New lagging suffix length is 12/15 for two/three +tabs. Final commit omission fails the ordinary durable/public comparison. + +The deferred-acquisition owner adds 32 finite histories over a real same-source +LEFT self-join, crossing subscriber/preload, one/two rows, literal/Promise +acquisitions, and settlement order. Every history requires an actual deferred +joined demand, checks zero acquisitions before its own reader, and checks +readiness and independently projected rows after successful coverage. Existing +production passes; retaining the parked participant makes all 32 fail readiness. +No production cycle was confirmed. Both full owners pass 540 tests. Browser +fixed/random actual overwrite reaches are repair 25/18 and callback 25/25. +These boundaries retain their controlled core and Node-host SQLite/Electric +limits. No HTTP, framework, browser-process, or arbitrary-shape closure follows. +See the [versioned amendment](oracle-reviews/2026-10-30-release-replay-retention.md#final-catch-up-provenance-and-deferred-sibling-resumption-review-2026-11-02). diff --git a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md index c0739316ea..e33ffe2478 100644 --- a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md +++ b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md @@ -443,3 +443,167 @@ No raw review item is deferred. The remaining limitations are the owners' explicit controlled and finite boundaries, not a known reachable violation of the repaired admission law. Final pushed CI and repair integration with actual main remain separate readiness obligations. No merge authorization follows. + + +## Final-catch-up provenance and deferred sibling resumption review (2026-11-02) + +This amendment evaluates CodeRabbit review **5738453581** on repair head +`b73f8e865bdda471df9dc77cc471e927143a41a9`. Its callback port starts from +`0b53825d4c19446fa3f346c1575820498804da9c`. The incremental run +`9a64a058-a898-4ac9-8d4c-4ecc87fdea30` selected eight files and skipped five +similar files. Selection does not establish eight fresh reviews. This is an +author self-review of the resulting oracle changes. No product law or shipped +source changes in this amendment. + +### Law, experiment, and inference + +**R1: eventual convergence needs an honest reach premise.** Browser owner Law 2 +and the coverage map promise the server head after every open stream catches +up. Durable/server divergence alone does not show that an older tab overwrote +a newer peer commit. Instrumenting the old grammar on unchanged production +recorded 58 final divergence repairs: 57 had no older-over-newer peer overwrite, +and one had it. The fixed campaign accepted 80 histories and initiated 81, +because the existing known-gap classifier discarded one history. These are +diagnostic counts, not a production RED or a new product defect. + +The owner now retains that old observation as `durable divergence repaired +during final catch-up`. Separate provenance records the delivered message's +server-log version, source handle, and committing tab. A differing older +same-key change must overwrite a newer peer commit. The marker remains until +the key reaches server head; a full snapshot resets this provenance. Final +rewrite reach counts only after the existing real durable/public comparison +passes. Provenance is independent model-side instrumentation over delivered +messages, not a production coordinator classification. + +The terminal dimension adds `lagging` beside the existing arbitrary, reset, +and stop endings. It starts a fresh handle, catches up every tab, establishes +the chosen key, catches up every tab again, makes two updates, lets a peer +commit both, and lets the selected tab commit only the older update. Final +catch-up must repair that actual overwrite. The new suffix has `6 + 3*tabCount` +steps (12 or 15). Reset/stop still add `4 + 2*tabCount` (8 or 10), and arbitrary +adds zero. The original 24-step stop prefix, arbitrary 8–24-step tail, model +predictions, comparisons, known gaps, 80-run budgets, and minima remain. + +The omission mutant suppresses only the final up-to-date commit while actual +older-over-newer overwrite evidence remains. It fails the ordinary durable and +public row comparison at `after final delivery to tab A` (v2 instead of v3). +It is an intended assertion failure, not a setup error or timeout. The captured +seed is `952051961`, shrink path `4:0:0:1:1:1:1:1:1:1:1:1:1:1:1:1`. +The shared direct replay interface reproduces that same mismatch. The oracle +file is restored after each temporary calibration. + +**R2: resumed deferred demand must progress after applied coverage.** Authority +is ARCHITECTURE law 15 (no acquisition before an own subscriber/preload), law 9 +(shared replay publication), and Initial demand (readiness after coverage). +The alleged completion cycle is refuted on its actual path. Existing +`startTruncateReplayDemand` removes `parkedParticipant` and decrements its +attempt before starting acquisition. The initial-result waiter can then wait +for completion without participating in the completion it awaits. + +The deferred-acquisition owner adds a finite 32-history grammar over a real +same-source LEFT self-join. It crosses subscriber/preload, one/two rows, +literal/held-Promise root and sibling results, and oldest/newest settlement. +The query starts its sync run without a reader and asserts zero acquisitions. +Its first source acquisition writes synchronously. This creates demand in the +second subscription while that subscription still defers acquisition and a +sibling replacement holds publication. A request-entry spy measures the actual +deferred-to-joined transition, with reach one required in every history. + +Held provider promises keep initial-query readiness pending. After controlled +successes, the query must be ready, any preload must fulfill, and public rows +must equal the independent self-join projection of input IDs. The driver allows +no later truncate, abort, window move, or extra preload to rescue the history. +All 32 pass against unchanged production on both branches. Leaving the parked +participant installed makes all 32 fail the readiness assertion with `loading` +instead of `ready`. Production bytes are restored afterward. The review's +major production repair is unnecessary; its durable value is this coverage. + +### Executed GREEN and calibration evidence + +On each branch, the complete deferred-acquisition owner passes **540 tests**, +including the new 32 histories. Both browser campaigns accept **80 histories** +per branch. The two existing final-delivery replays pass on both branches. +Repair ran its missing second replay separately after the first selector chose +only three tests. Callback ran all four tests in one invocation. + +| Branch/campaign | Final stopped-source restart | Durable divergence repair | Actual older-peer overwrite repair | Reset-abandoned final recovery | +| --- | ---: | ---: | ---: | ---: | +| Repair fixed | 36 | 63 | 25 | 16 | +| Repair random | 34 | 60 | 18 | 25 | +| Callback fixed | 36 | 63 | 25 | 16 | +| Callback random | 38 | 66 | 25 | 18 | + +Earlier records describing 46, 52, or 58-style final counts as lagging rewrites +are historical divergence counters. They do not establish older-over-newer +peer overwrite coverage. This correction preserves their original records and +does not retroactively strengthen that evidence. + +Focused core/browser TypeScript, ESLint, formatting, whitespace, and +`pnpm lint:changed` validate the follow-up. Callback's first browser TypeScript +attempt lacked local module paths. Temporary links to already installed cached +modules repair setup and are removed afterward. No dependency install, manifest +or lockfile change occurs. Local Vitest is cached 3.2.4; CI uses 4.1.12. Full +repository tests and builds are not run for this oracle-only amendment. + +### Grammar controls and oracle self-review + +Reconstruction: the old arbitrary/reset/stop regimes remain reconstructible. +The lagging suffix reconstructs an actual older commit over a newer peer and +leaves it for the final comparison. The deferred grammar reconstructs the +reviewed parked demand during real sibling resumption. Ablation: remove either +update, the peer commit, or the selected older commit and the overwrite premise +is lost. Remove the deferred start or synchronous first delivery and the parked +sibling path is lost. Promise axes distinguish pending readiness from literal +completion; settlement order distinguishes the two active acquisition gates. + +Range: browser histories retain two/three tabs, keys 1–3, every existing stop +kind and tail action, plus all four endings. Deferred histories bound one/two +rows and two source subscriptions with both public initiating paths. Zero rows +cannot create the reported keyed sibling demand and is outside this lane; the +existing broad owner retains empty/no-provider regimes. Exclusion: the suffix +commits an update only after establishing its key; it never fabricates an +update to a missing row. The deferred lane never starts provider work before +the first reader and never settles an acquisition that was not issued. + +| Requirement | Outcome for this amendment | +| --- | --- | +| ORC-001 | Existing Law 2 and ARCHITECTURE laws 9/10/15 authorize results. Limits below remain explicit. | +| ORC-002 | Input-ID self-join recomputation and server-message commit provenance use no production semantic classifier. | +| ORC-003 | Contract, model, histories, driver, and checkpoints remain visible beside executable code. | +| ORC-004 | Reconstruction, ablation, range, and exclusion appear above. | +| ORC-005 | Real core live-query Collection and real SQLite/coordinator paths compare readiness, rows, and durable/public state. Supplied messages contain the source facts needed at each cut. | +| ORC-006 | Parked-participant and actual-final-commit omissions fail their intended assertions; neither is a setup failure. | +| ORC-007 | Browser fixed/random campaigns use the same property, grammar, recorder, comparison, and 80-run budget. Direct seed/path mutant replay reproduces the row mismatch. The 32-case finite enumeration needs no random campaign. | +| ORC-008 | Provenance is reach instrumentation, not a change to expected product state. Equal row divergence can have different commit origins, so provenance distinguishes which coverage claim applies. | +| ORC-009 | Comments map provenance to delivered message position/tab/handle and distinguish demand from acquisition and completion. | +| ORC-010 | Existing browser failure/cleanup handling remains. Calibration scripts restore exact bytes in finally. The deferred driver restores its spy, settles held gates, and cleans Collections. | +| ORC-011 | No shared semantic-fault hypothesis needs a second formulation. The independent self-join result and explicit commit provenance address the two named hypotheses. | +| ORC-012 | This versioned record binds the reviewed parent heads, evidence, dispositions, and every applicable requirement. | +| ORC-013 | Divergence without an older overwrite is the neighboring false-positive case. Literal versus held results and both settlement orders distinguish readiness boundaries. | +| ORC-014 | Claims stop at core controlled acquisition and Node SQLite/coordinator controlled Electric messages. No HTTP provider, framework, or browser-process handoff claim follows. | +| ORC-015 | Browser reach requires each retained minimum plus honest divergence and actual overwrite states. Every deferred history requires one actual deferred joined demand. | +| ORC-016 | Histories share independent projection/comparison or the existing generated driver. No standalone fixed expected rows or type assertions enter an oracle. | + +### Loss audit, reviewer assessment, and readiness + +The two inline findings are fully accounted: **one fixed-now** (R1 test +integrity), **one refuted** (R2 production cycle), with its missing receiving +history added. No deferral or open in-scope finding remains from this review. +The raw audit also retains summary/risk duplicates, five related-issue links, +the informational docstring metric (0%, one function, four files, four +unsupported), run metadata, and generator/autofix suggestions. No boilerplate +command runs. Related links do not establish issue closure. + +The reviewer earns a qualified hire recommendation for oracle scrutiny. R1 +correctly distinguishes a classifier's premise from ordinary convergence. R2 +identifies a useful lifecycle intersection but overlooks the existing participant +removal, so its major severity and proposed production fix lack support. Use +runtime witnesses before relying on this reviewer for lifecycle merge decisions. + +Evidence covers finite core resumption and controlled-stream Node persistence +histories at named cuts. It does not prove arbitrary provider failures, +framework schedules, browser processes, every join shape, or the existing +excluded/pinned known gaps. No new product behavior or shipped source changes. +Current-head CI after the oracle commits and actual-main repair integration +remain separate readiness requirements. No merge or publication is authorized +by this review. diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 6b3c3dd7cf..2f304b9ef5 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -1099,3 +1099,25 @@ The shared runtime, primary and browser hashes match the release amendment's final binding table. Compiler and Solid source/test paths remain exactly those of a2e75c3. The containing commit binds this port; full commit IDs and changed path hashes are retained in scratch final-binding.json and the PR body. + + +## Final-catch-up provenance and deferred sibling resumption port (2026-11-02) + +The follow-up on callback parent `0b53825d4c19446fa3f346c1575820498804da9c` +ports the oracle-only evaluation of CodeRabbit review 5738453581 from repair +parent `b73f8e865bdda471df9dc77cc471e927143a41a9`. No shipped source changes. +The shared [versioned amendment](2026-10-30-release-replay-retention.md#final-catch-up-provenance-and-deferred-sibling-resumption-review-2026-11-02) +records authority, grammar controls, calibrations, all ORC outcomes and limits. + +The browser reach now distinguishes ordinary durable/server divergence from +an actual older same-key commit over a newer peer. Fixed/random 80-history +campaigns pass: final restart 36/38, divergence repair 63/66, actual overwrite +repair 25/25, and reset-abandoned final recovery 16/18. Both final-delivery +replays pass. The deferred-acquisition owner passes all 540 tests, including +32 same-path sibling-resumption histories with reach one per history. Existing +production finishes them; a retained parked participant fails readiness in all +32 calibration cases on the shared runtime. The production-cycle finding is +refuted, with its useful coverage retained. Historical final rewrite counts +remain divergence measurements. Compiler, Solid, browser runtime, and core +runtime sources stay exact to the parent. CI after this oracle commit remains +pending; paused callback review at 8464 supplies no current-head review. diff --git a/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts index 1dc3261503..11c87c0109 100644 --- a/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/multi-tab-electric-oracle.test.ts @@ -975,7 +975,8 @@ describe(`multi-tab Electric over one persisted database`, () => { // of an initially absent key restore the original rows before the tail. // The prefix adds 24 steps to the unchanged 8–24-step arbitrary tail. An // ending dimension either leaves that final state untouched or adds a - // constructive reset/stop suffix of 4 + 2*tabCount steps (8 or 10). Removing + // reset/stop suffix of 4 + 2*tabCount steps (8 or 10), or a lagging-commit + // suffix of 6 + 3*tabCount steps (12 or 15). Removing // the open delivery, stop, or peer commit loses the law's premise; recovering // before the peer commit would test a different boundary. Existing pinned // histories still use the same driver without this prefix. @@ -989,6 +990,7 @@ describe(`multi-tab Electric over one persisted database`, () => { `arbitrary` as const, `reset` as const, `stop` as const, + `lagging` as const, ), stopOrder: fc.shuffledSubarray([0, 1, 2, 3], { minLength: 4, @@ -1026,8 +1028,13 @@ describe(`multi-tab Electric over one persisted database`, () => { // without committing it. Reset abandons that reservation, so final // delivery judges its failure before recovery (Laws 2/5). Stop // instead leaves final catch-up to restart the stopped source. - // The source tab, key and two/three-tab host vary with this grammar. - // Removing advance or reset loses that premise; catching up here + // The lagging ending first establishes the key, then a peer commits + // two updates and the selected tab commits only the older one. It + // leaves that actual overwrite for final catch-up to repair. A + // server change alone establishes only durable/server divergence. + // Tab roles, key and two/three-tab host vary with this grammar. + // Removing either update or the partial commit loses the overwrite; + // removing advance or reset loses that premise; catching up here // would move the failure out of the final observation cut. ...(ending === `arbitrary` ? [] @@ -1040,11 +1047,26 @@ describe(`multi-tab Electric over one persisted database`, () => { { kind: `catch up`, tab: index, pick: 0 }, ], ).flat(), - { kind: `server change` as const, key, pick: 0 }, - { kind: `advance` as const, tab, pick: 0 }, - ending === `reset` - ? { kind: `peer reset` as const } - : { kind: `stop` as const, tab, pick: stopOrder[0]! }, + ...(ending === `lagging` + ? [ + { kind: `server change` as const, key, pick: 1 }, + ...Array.from({ length: tabCount }, (_, index) => ({ + kind: `catch up` as const, + tab: index, + pick: 0, + })), + { kind: `server change` as const, key, pick: 1 }, + { kind: `server change` as const, key, pick: 1 }, + { kind: `catch up` as const, tab: peer, pick: 0 }, + { kind: `catch up` as const, tab, pick: 1 }, + ] + : [ + { kind: `server change` as const, key, pick: 0 }, + { kind: `advance` as const, tab, pick: 0 }, + ending === `reset` + ? { kind: `peer reset` as const } + : { kind: `stop` as const, tab, pick: stopOrder[0]! }, + ]), ]), ], } @@ -1162,13 +1184,52 @@ describe(`multi-tab Electric over one persisted database`, () => { let serverVersion = 0 generatedReach.beginHistory() + // Reach provenance comes from the server messages the driver delivered, + // not merely a difference between durable rows and the current server. + // A newer peer commit must be overwritten by an older change to that key. + // This model-only provenance is a delivered message's server position and + // committing tab. Snapshot rows start at version zero of their handle; + // change messages retain their exact log version, including deletions. + const durableOrigins = new Map< + number, + { + tab: TabModel + handle: number | undefined + version: number + } + >() + const laggingOverwrites = new Set() const commit = (model: TabModel) => { - if (model.truncates) durable.clear() + if (model.truncates) { + durable.clear() + durableOrigins.clear() + laggingOverwrites.clear() + } for (const message of model.open ?? []) { if (!(`key` in message)) continue const row = message.value + const version = + log.changes.find((change) => change.message === message)?.version ?? 0 + const previous = durableOrigins.get(row.id) + const value = + message.headers.operation === `delete` ? undefined : row.label + if ( + previous && + previous.tab !== model && + previous.handle === model.handle && + previous.version > version && + durable.get(row.id) !== value + ) + laggingOverwrites.add(row.id) if (message.headers.operation === `delete`) durable.delete(row.id) else durable.set(row.id, row.label) + durableOrigins.set(row.id, { + tab: model, + handle: model.handle, + version, + }) + if (durable.get(row.id) === log.rows.get(row.id)) + laggingOverwrites.delete(row.id) } model.open = undefined model.truncates = false @@ -1543,20 +1604,30 @@ describe(`multi-tab Electric over one persisted database`, () => { generatedReach.reach(`stopped source restarted during final catch-up`) await recover(index, `restart`) } - const laggingBeforeFinal = + const overwrittenBeforeFinal = [...laggingOverwrites] + const divergentBeforeFinal = rowsOf(durable).some(({ id, label }) => log.rows.get(id) !== label) || rowsOf(log.rows).some(({ id, label }) => durable.get(id) !== label) await deliver(index, `all`, true) + // A reset-abandoned transaction fails only when delivery tries to + // commit it. Judge Law 5 at that cut before the restart establishes + // Law 2's successful catch-up premise. + await check(`after final delivery to tab ${tabs[index]!.tab}`) if ( - laggingBeforeFinal && + divergentBeforeFinal && !models[index]!.failedByReset && JSON.stringify(rowsOf(durable)) === JSON.stringify(rowsOf(log.rows)) + ) + generatedReach.reach( + `durable divergence repaired during final catch-up`, + ) + if ( + overwrittenBeforeFinal.some( + (id) => + !laggingOverwrites.has(id) && durable.get(id) === log.rows.get(id), + ) ) generatedReach.reach(`lagging commit rewritten during final catch-up`) - // A reset-abandoned transaction fails only when delivery tries to - // commit it. Judge Law 5 at that cut before the restart establishes - // Law 2's successful catch-up premise. - await check(`after final delivery to tab ${tabs[index]!.tab}`) if (models[index]!.failedByReset) { await recover(index, `restart`) await deliver(index, `all`, true) @@ -1741,6 +1812,7 @@ describe(`multi-tab Electric over one persisted database`, () => { type GeneratedReach = | `stopped source restarted during final catch-up` + | `durable divergence repaired during final catch-up` | `lagging commit rewritten during final catch-up` | `must-refetch delivered` | `peer commit during an open reserved transaction` @@ -1765,6 +1837,7 @@ describe(`multi-tab Electric over one persisted database`, () => { const createGeneratedReach = () => createOracleReach(`browser.multi-tab-reserved-history`, [ `stopped source restarted during final catch-up`, + `durable divergence repaired during final catch-up`, `lagging commit rewritten during final catch-up`, `must-refetch delivered`, `peer commit during an open reserved transaction`, diff --git a/packages/db/tests/live-query-deferred-acquisition-oracle.test.ts b/packages/db/tests/live-query-deferred-acquisition-oracle.test.ts index 9b29526b7a..b549f74352 100644 --- a/packages/db/tests/live-query-deferred-acquisition-oracle.test.ts +++ b/packages/db/tests/live-query-deferred-acquisition-oracle.test.ts @@ -41,18 +41,25 @@ * Its admission releases that deferral. A failed sync start remains an error * in that run; releasing the old render handle does not implicitly retry it. * - * Limits: source Collections load synchronously, so every checkpoint is - * settled. The grammar crosses seven source states, two query shapes, and one or + * Limits: the broad grammar uses synchronous source loads and settled + * checkpoints. It crosses seven source states, two query shapes, and one or * two live-query levels, with one peer consumer and one truncate before the - * first subscriber. It does not generate source failure, asynchronous loads, - * or several live-query consumers; the subscription lifecycle oracle owns - * those once acquisition has resumed. Pinned blocks cover pooled views, + * first subscriber. The separate sibling-resumption grammar below uses + * controlled asynchronous loads. The broad grammar does not generate source + * failure, asynchronous loads, or several live-query consumers. The + * subscription lifecycle oracle owns those once acquisition has resumed. Pinned blocks cover pooled views, * `DbClient` stream preloads, a subscriber that outlives cleanup, and reads * that wait for readiness. */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { fc } from '@fast-check/vitest' -import { createCollection } from '../src' +import { + BTreeIndex, + createCollection, + createLiveQueryCollection, + eq, +} from '../src' +import { CollectionSubscription } from '../src/collection/subscription.js' import { resumeDeferredLiveQueryCollections } from '../src/live-query-options.js' import { Query } from '../src/query/builder/index.js' import { createPooledLiveQuery } from '../src/query/pooled-live-query.js' @@ -435,8 +442,13 @@ describe(`live-query deferred acquisition`, () => { */ // The pooled entry point takes the internal builder, as its own oracle does. declareOracle({ - check: [runAcquisitionHistory, runPooledHistory], - model: [initialModel, applyModel, expectedReplacementRun], + check: [runAcquisitionHistory, runPooledHistory, runDeferredSiblingHistory], + model: [ + initialModel, + applyModel, + expectedReplacementRun, + deferredSiblingRows, + ], }) async function runPooledHistory( @@ -747,3 +759,168 @@ describe(`render sync-start handoff error provenance`, () => { }) } }) + +/** + * A first subscriber resumes source subscriptions in registration order. The + * first source load can synchronously deliver rows that create a lazy demand + * in another, still-deferred subscription to that same source. Law 15 keeps + * that demand detached until its own resumption; laws 9 and 10 require the + * completed query once every resumed load has applied. A sibling's publication + * and the initial-query waiter must not wait for one another in a cycle. + * + * This finite grammar crosses subscriber/preload, one/two rows, and literal or + * held Promise results for each acquisition, and both settlement orders. + * It uses the real self-join builder, without a supplied publication-control + * callback. Request instrumentation records the + * actual deferred-to-joined transition; it supplies no expected row values. + * The independent self-join law below retains each input row exactly once. + * No provider or framework scheduling promise follows from this core path. + */ +type DeferredSiblingHistory = { + reader: `subscribe` | `preload` + rowCount: 1 | 2 + rootAsync: boolean + siblingAsync: boolean + newestFirst: boolean +} +const deferredSiblingHistories: Array = ( + [`subscribe`, `preload`] as const +).flatMap((reader) => + ([1, 2] as const).flatMap((rowCount) => + [false, true].flatMap((rootAsync) => + [false, true].flatMap((siblingAsync) => + [false, true].map((newestFirst) => ({ + reader, + rowCount, + rootAsync, + siblingAsync, + newestFirst, + })), + ), + ), + ), +) +function deferredSiblingRows(rows: ReadonlyArray<{ id: string }>) { + return rows.map(({ id }) => ({ id, peer: id })) +} +async function runDeferredSiblingHistory(history: DeferredSiblingHistory) { + const rows = Array.from({ length: history.rowCount }, (_, i) => ({ + id: `row-${i}`, + })) + const gates: Array<() => void> = [] + let loads = 0 + let wrote = false + const source = createCollection<{ id: string }>({ + getKey: (row) => row.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + gcTime: 0, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: () => { + loads++ + if (!wrote) { + wrote = true + begin() + for (const row of rows) + write({ type: `insert`, value: { ...row } }) + commit() + } + if (!(loads === 1 ? history.rootAsync : history.siblingAsync)) + return true + return new Promise((resolve) => gates.push(resolve)) + }, + } + }, + }, + }) + const query = createLiveQueryCollection({ + query: (q) => + q + .from({ left: source }) + .leftJoin({ right: source }, ({ left, right }) => eq(left.id, right.id)) + .select(({ left, right }) => ({ id: left.id, peer: right.id })), + gcTime: 0, + }) + const reach = createOracleReach(`deferred-sibling`, [ + `deferred joined demand`, + ]) + reach.beginHistory() + const original = CollectionSubscription.prototype.requestSnapshot + const spy = vi + .spyOn(CollectionSubscription.prototype, `requestSnapshot`) + .mockImplementation(function (this: CollectionSubscription, ...args) { + const deferred = this.isDeferringAcquisition() + const isPending = () => this.hasPendingTruncateReplacement + const before = isPending() + const result = original.apply(this, args) + if (deferred && !before && isPending()) + reach.reach(`deferred joined demand`) + return result + }) + let subscription: ReturnType | undefined + let preloadSettled = false + let preloadFailure: unknown + try { + query.startSyncImmediate() + await settle() + expect(loads, `deferred query starts no acquisition`).toBe(0) + if (history.reader === `subscribe`) + subscription = query.subscribeChanges(() => {}) + else + void query.preload().then( + () => { + preloadSettled = true + }, + (error: unknown) => { + preloadFailure = error + }, + ) + // The provider promises are semantic gates. Drain every issued gate, + // including any continuation produced by their settlement. No truncate, + // abort, window move or extra preload is allowed to repair this history. + await settle() + if (history.rootAsync || history.siblingAsync) { + expect(query.status, `held acquisition gates initial readiness`).toBe( + `loading`, + ) + expect(preloadSettled).toBe(false) + } + for (let turn = 0; turn < 12; turn++) { + const resolve = history.newestFirst ? gates.pop() : gates.shift() + resolve?.() + await settle() + } + console.info( + `deferred-sibling reach`, + history, + Object.fromEntries(reach.counts()), + ) + reach.assertReached() + expect(loads, `each source subscription acquires once`).toBe(2) + expect(query.status, `resumed sibling replay must finish`).toBe(`ready`) + if (history.reader === `preload`) { + expect(preloadFailure).toBeUndefined() + expect(preloadSettled).toBe(true) + } + expect( + query.toArray + .map(({ id, peer }) => ({ id, peer })) + .sort((a, b) => a.id.localeCompare(b.id)), + ).toEqual(deferredSiblingRows(rows)) + } finally { + spy.mockRestore() + for (const resolve of gates.splice(0)) resolve() + subscription?.unsubscribe() + await query.cleanup() + await source.cleanup() + } +} +describe(`deferred demand joining a sibling replay`, () => { + for (const history of deferredSiblingHistories) + it(`finishes after resumption: ${JSON.stringify(history)}`, () => + runDeferredSiblingHistory(history)) +}) From ee4e98db993b9b393dfac1744795512716ffe790 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 08:37:08 -0700 Subject: [PATCH 18/19] docs: clarify controlled replay evidence scope --- .../oracle-reviews/2026-10-30-release-replay-retention.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md index e33ffe2478..7e274643e2 100644 --- a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md +++ b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md @@ -580,7 +580,7 @@ the first reader and never settles an acquisition that was not issued. | ORC-011 | No shared semantic-fault hypothesis needs a second formulation. The independent self-join result and explicit commit provenance address the two named hypotheses. | | ORC-012 | This versioned record binds the reviewed parent heads, evidence, dispositions, and every applicable requirement. | | ORC-013 | Divergence without an older overwrite is the neighboring false-positive case. Literal versus held results and both settlement orders distinguish readiness boundaries. | -| ORC-014 | Claims stop at core controlled acquisition and Node SQLite/coordinator controlled Electric messages. No HTTP provider, framework, or browser-process handoff claim follows. | +| ORC-014 | Claims cover controlled core acquisition. The SQLite/coordinator tests run in Node with controlled Electric messages. No HTTP provider, framework, or browser-process handoff claim follows. | | ORC-015 | Browser reach requires each retained minimum plus honest divergence and actual overwrite states. Every deferred history requires one actual deferred joined demand. | | ORC-016 | Histories share independent projection/comparison or the existing generated driver. No standalone fixed expected rows or type assertions enter an oracle. | From e9864bc7157c378dec2ade26deef4309b178f5ba Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 2 Nov 2026 09:20:27 -0700 Subject: [PATCH 19/19] refactor(db): share smaller replay acquisition paths --- .../2026-10-30-release-replay-retention.md | 52 +++++++ .../oracle-reviews/callback-metadata.md | 12 ++ packages/db/src/collection/subscription.ts | 144 ++++++++---------- 3 files changed, 126 insertions(+), 82 deletions(-) diff --git a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md index 7e274643e2..4883325cb6 100644 --- a/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md +++ b/docs/contributing/oracle-reviews/2026-10-30-release-replay-retention.md @@ -607,3 +607,55 @@ excluded/pinned known gaps. No new product behavior or shipped source changes. Current-head CI after the oracle commits and actual-main repair integration remain separate readiness requirements. No merge or publication is authorized by this review. + +## Compressed-size refactor on current main (2026-11-02) + +The maintainer requested negative compressed bundle size for this repair. Both +branches now include main `035a5260ef994ce3aabed77a3f09b35c71482a9a` through normal +merge commits. Their merged trees exactly matched the preceding clean scratch +integration trees before this refactor. + +The subscription constructs its acquisition record once, including for detached +demand. It captures the replay attempt once, uses the existing attempt finisher +for its setup hold, and shares the existing uncaught-error reporter. Replay +failure collection retains the same membership checks and Map insertion order +without intermediate filter/map arrays. Acquisition admission has one result +return after its cancellation, release, or participation branch. The replay +factory reads the current sync-run generation itself instead of receiving the +same value from every caller. + +This is an internal refactor under architecture laws 9, 10, 15 and 16. The +accepted behavior, oracle models, history grammars, comparisons, reach minima +and run budgets do not change. The original production RED and hostile-mutant +evidence above remains historical. This refactor introduces no new behavior or +bug-class closure claim, so it uses existing applicable owners without changing +their expectations. + +On each branch, one final-source run passes 1,787 tests in 11 files: replay +refinement, its fixed neighbors, deferred acquisition, lifecycle history and +publication, subscription replay, lifecycle ownership and cells, adapter +ownership, preload indexing and index reuse. Focused TypeScript, ESLint, +formatting, whitespace and `pnpm lint:changed` pass. Each branch uses one Vitest +worker, for two total. Local Vitest is cached 3.2.4, while CI uses 4.1.12. No +install, lockfile edit, full repository build or full `pnpm test` run occurs. + +The existing `pnpm size:compare --base 035a5260ef994ce3aabed77a3f09b35c71482a9a +--exact-base --packages db --files` command measures the targeted production +build and app scenarios. The repair's package gzip changes from 109,825 to +109,824 bytes, a decrease of 1 byte. Its package Brotli decreases by 114 bytes. +Its typical app gzip changes from 96,410 to 96,399 bytes, a decrease of 11 bytes. +Every measured app scenario decreases with gzip. Typical app Brotli increases +by 14 bytes, and some other Brotli scenarios increase. The package-only `--dev` +measure, which includes development code as CI does, decreases by 2 gzip bytes +and 19 Brotli bytes. These local measures do not replace the remote size check. + +The callback port's package gzip decreases by 59 bytes and package Brotli by +38 bytes against the same main. Its typical app gzip decreases by 75 bytes. +Typical app Brotli increases by 26 bytes. Its earlier size figures retain their +original historical bindings. + +The finite core histories and current-main indexing checks supply the bounds +of this verification. Existing provider, framework, browser-process and deeper +history limits remain. Browser and Solid receiving evidence above retains its +historical source binding. Final-head CI and external review coverage remain +separate from these local results. diff --git a/docs/contributing/oracle-reviews/callback-metadata.md b/docs/contributing/oracle-reviews/callback-metadata.md index 2f304b9ef5..31195cfb37 100644 --- a/docs/contributing/oracle-reviews/callback-metadata.md +++ b/docs/contributing/oracle-reviews/callback-metadata.md @@ -1121,3 +1121,15 @@ refuted, with its useful coverage retained. Historical final rewrite counts remain divergence measurements. Compiler, Solid, browser runtime, and core runtime sources stay exact to the parent. CI after this oracle commit remains pending; paused callback review at 8464 supplies no current-head review. + +## Shared compressed-size refactor (2026-11-02) + +The callback branch includes main `035a5260ef994ce3aabed77a3f09b35c71482a9a` +through a normal merge and carries the identical subscription refactor as the +release repair. The [compressed-size record](2026-10-30-release-replay-retention.md#compressed-size-refactor-on-current-main-2026-11-02) +records the changes, exact baseline, final tests and limits. Both branches pass +1,787 tests in 11 unchanged owners on their final sources. The callback package +decreases by 59 gzip bytes and 38 Brotli bytes. Its typical app decreases by +75 gzip bytes and increases by 26 Brotli bytes. Earlier source fingerprints and +size figures remain historical. No oracle law, comparison, grammar or budget +changes. Fresh CI must check this pushed head before merge. diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 552c2f96c5..aaf45c0cf1 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -447,7 +447,7 @@ export class CollectionSubscription const joins = open?.syncRunGeneration === syncRunGeneration const replayState = joins ? open - : this.createTruncateReplayState(syncRunGeneration, () => { + : this.createTruncateReplayState(() => { const currentRows = this.collection.currentStateAsChanges({ optimizedOnly: false, }) @@ -510,7 +510,6 @@ export class CollectionSubscription } else { // Every overlapping attempt shares one publication baseline and buffer. replayState = this.createTruncateReplayState( - this.collection._sync.getSyncRunGeneration(), () => new Map(this.publishedRows), ) this.truncateReplayState = replayState @@ -561,11 +560,9 @@ export class CollectionSubscription attempt: TruncateReplayAttempt, demand: SubsetDemand, ): void { - const isCurrentAttempt = () => - this.truncateReplayState === replayState && - replayState.currentAttempt === attempt const isCurrent = () => - isCurrentAttempt() && + this.truncateReplayState === replayState && + replayState.currentAttempt === attempt && this.isSyncRunGenerationCurrent(replayState.syncRunGeneration) && this.isDemandActive(demand) const waiting = demand.initialResult @@ -590,7 +587,7 @@ export class CollectionSubscription if (hadPreviousAcquisition) this.releaseAcquisition(previous) if (!isCurrent() || demand.requestOptions.signal?.aborted) return - const next = this.createSubsetAcquisitionRecord(demand) + const next = this.createSubsetAcquisitionRecord(demand.requestOptions) next.attempt = attempt demand.acquisition = next demand.acquisitionState = `starting` @@ -637,9 +634,7 @@ export class CollectionSubscription // Replay settlement runs from a Promise callback, so throwing here would // create an unobserved derived rejection. Surface subscriber errors like // other async collection events instead. - queueMicrotask(() => { - throw error - }) + reportUncaught(error) } } @@ -714,18 +709,19 @@ export class CollectionSubscription if (replayState.pendingSetups > 0 || replayState.pending.size > 0) return // A held replacement that loaded decides the demands it replaces. - const activeFailures = [...replayState.failures] - .filter( - ([demand]) => - this.subsetDemands.includes(demand) && - !this.subsetDemands.some( - (replacement) => - replacement.replaces?.includes(demand) && - !replayState.failures.has(replacement) && - !replayState.unloaded.has(replacement), - ), + const activeFailures: Array = [] + for (const [demand, error] of replayState.failures) { + if ( + this.subsetDemands.includes(demand) && + !this.subsetDemands.some( + (replacement) => + replacement.replaces?.includes(demand) && + !replayState.failures.has(replacement) && + !replayState.unloaded.has(replacement), + ) ) - .map(([, error]) => error) + activeFailures.push(error) + } try { if (activeFailures.length > 0) { this.abandonTruncateReplay(replayState, activeFailures) @@ -880,10 +876,7 @@ export class CollectionSubscription */ failReplacement(error: Error): void { if (this.truncateReplayState) return - const replayState = this.createTruncateReplayState( - this.collection._sync.getSyncRunGeneration(), - () => new Map(), - ) + const replayState = this.createTruncateReplayState(() => new Map()) replayState.pendingSetups = 0 replayState.failed = true replayState.completion.reject(error) @@ -933,11 +926,10 @@ export class CollectionSubscription /** One replay state; only direct subscribers buffer a private replacement. */ private createTruncateReplayState( - syncRunGeneration: number, privateRows: () => Map, ): TruncateReplayState { return { - syncRunGeneration, + syncRunGeneration: this.collection._sync.getSyncRunGeneration(), publicationState: { loadedInitialState: this.loadedInitialState, snapshotSent: this.snapshotSent, @@ -1125,18 +1117,19 @@ export class CollectionSubscription /** Create the record for a fresh, abortable acquisition attempt. */ private createSubsetAcquisitionRecord( - demand: SubsetDemand, + requestOptions: LoadSubsetOptions, + detached = false, ): SubsetAcquisitionRecord { - // Eager sync never passes subset options to an adapter, so an abortable - // acquisition would only allocate a controller and an AbortError. - if (this.collection.config.syncMode !== `on-demand`) { + // Detached demand has no physical work. Eager sync never passes subset + // options to an adapter. Neither needs an abortable acquisition yet. + if (detached || this.collection.config.syncMode !== `on-demand`) { return { - options: demand.requestOptions, + options: requestOptions, syncRunGeneration: this.collection._sync.getSyncRunGeneration(), } } const abortController = new AbortController() - const requestSignal = demand.requestOptions.signal + const requestSignal = requestOptions.signal let removeRequestAbortListener: (() => void) | undefined if (requestSignal?.aborted) { @@ -1150,7 +1143,7 @@ export class CollectionSubscription return { options: { - ...demand.requestOptions, + ...requestOptions, signal: abortController.signal, }, syncRunGeneration: this.collection._sync.getSyncRunGeneration(), @@ -1193,28 +1186,14 @@ export class CollectionSubscription !this.truncateReplayState && this.options.truncateReplayPublication?.siblingReplays() ) { - joinedReplay = this.createTruncateReplayState( - this.collection._sync.getSyncRunGeneration(), - () => new Map(), - ) + joinedReplay = this.createTruncateReplayState(() => new Map()) this.truncateReplayState = joinedReplay this.truncateReplacementPending = true } - const joinedAttempt = joinedReplay?.currentAttempt + const replayState = this.truncateReplayState + const replayAttempt = replayState?.currentAttempt try { - const demand: SubsetDemand = { - requestOptions, - acquisition: { - options: requestOptions, - syncRunGeneration: this.collection._sync.getSyncRunGeneration(), - }, - acquisitionState: `starting`, - } - // A demand that starts during an open replay joins it, parked or not (law - // 9: a new demand joins the open replay). - const replayState = this.truncateReplayState - const replayAttempt = replayState?.currentAttempt - if ( + const detached = this.collection.status === `cleaned-up` || // A deferring subscription starts no acquisition attempt until it resumes. (this.defersAcquisition && @@ -1225,9 +1204,21 @@ export class CollectionSubscription (isFailedStatus(this.collection.status) || (this.collection.status !== `idle` && this.collection._sync.syncLoadSubsetFn === null))) - ) { - demand.acquisitionState = `detached` - this.subsetDemands.push(demand) + const acquisition = this.createSubsetAcquisitionRecord( + requestOptions, + detached, + ) + const demand: SubsetDemand = { + requestOptions, + acquisition, + acquisitionState: detached ? `detached` : `starting`, + } + // A demand that starts during an open replay joins it, parked or not (law + // 9: a new demand joins the open replay). + // Reentrant release must see the exact acquisition before adapter work + // starts. A load that throws is a rejection: it keeps its owner. + this.subsetDemands.push(demand) + if (detached) { const initialResult = createDeferred() demand.initialResult = initialResult const abort = () => @@ -1248,44 +1239,33 @@ export class CollectionSubscription ) return { demand, result: initialResult.promise, started: false } } - const acquisition = this.createSubsetAcquisitionRecord(demand) - demand.acquisition = acquisition - const syncRunGeneration = this.collection._sync.getSyncRunGeneration() - // Reentrant release must see the exact acquisition before adapter work - // starts. A load that throws is a rejection: it keeps its owner. - this.subsetDemands.push(demand) + const syncRunGeneration = acquisition.syncRunGeneration const result = this.collection._sync.loadSubset(acquisition.options) if (!this.isSyncRunGenerationCurrent(syncRunGeneration)) { const demandIndex = this.subsetDemands.indexOf(demand) if (demandIndex !== -1) this.subsetDemands.splice(demandIndex, 1) cancelAcquisition(acquisition) - return { demand, result, started: true } - } - - demand.acquisitionState = `active` - if (!this.subsetDemands.includes(demand)) { - this.releaseAcquisition(acquisition) - return { demand, result, started: true } - } - - if (replayState && replayAttempt) { - this.trackTruncateReplayParticipant( - replayState, - replayAttempt, - demand, - result, - acquisition.options.signal, - ) + } else { + demand.acquisitionState = `active` + if (!this.subsetDemands.includes(demand)) { + this.releaseAcquisition(acquisition) + } else if (replayState && replayAttempt) { + this.trackTruncateReplayParticipant( + replayState, + replayAttempt, + demand, + result, + acquisition.options.signal, + ) + } } return { demand, result, started: true } } finally { // Adapter and status callbacks can reenter before returning a promise. // Setup holds the replacement until that tentative work is registered. - if (joinedReplay && joinedAttempt) { - joinedReplay.pendingSetups-- - joinedAttempt.setupComplete = true - this.checkTruncateReplayComplete(joinedReplay) + if (joinedReplay && replayAttempt) { + this.startTruncateReplayAttempt(joinedReplay, replayAttempt, []) } } }