diff --git a/.github/workflows/formal-full.yaml b/.github/workflows/formal-full.yaml index 2fa711f3..b583b380 100644 --- a/.github/workflows/formal-full.yaml +++ b/.github/workflows/formal-full.yaml @@ -70,12 +70,17 @@ jobs: # challenge catalog alone, so a port-only change cannot alter it. if: (github.event_name != 'schedule' || github.ref == 'refs/heads/main') && inputs.mode != 'replay' && inputs.mode != 'mutations' runs-on: ubuntu-latest - # Shared-library challenges now execute the importing profiles' partitions, - # including clean baselines and mutant checks cached by exact source closure. - # The expanded partition campaign was still running after an hour in - # run 35568655690. Leave room for the full campaign on slower runners - # without reducing models, histories or partitions. - timeout-minutes: 150 + # make formal-check runs every scheduled model, measures every catalogued + # model fault on four Quint workers, then checks the lint baseline and the + # kernel fixtures. Run 39239127644 on main spent 6.0, 134.6 and 2.3 + # minutes on them, its 79 challenges 528 worker-minutes in their logged + # phases. Run 40245022083 reached the former 150-minute timeout with 78 of + # this catalog's 90 challenges done at 7.1 worker-minutes each; the 12 left + # put the campaign near 156 minutes. typescript/test/formal-validation.test.ts + # pins the challenges the timeout holds, so catalog growth fails the pull + # request until the timeout grows rather than reducing models, histories + # or partitions. + timeout-minutes: 210 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - uses: ./.github/actions/setup-validation @@ -205,9 +210,11 @@ jobs: # mutants: about 1 minute per TypeScript mutant on a fast runner and about 2 # on the slow class (13 mutants took 17 minutes unsharded in run # 34669546872). typescript/test/formal-validation.test.ts pins the mutants - # a shard may hold so that the baselines (2-4 minutes), the slow-class cost and - # one hung cohort's 9-minute bound fit the timeout; growing the catalog past - # that fails the pull request until the matrix grows. + # a shard may hold so that setup and the baselines (1.0 to 1.6 minutes in + # the 18 shards of runs 40142189738, 40146831443 and 40245022083; the pin + # allows 2), the slow-class cost and one hung cohort's 9-minute bound fit + # the timeout; growing the catalog past that fails the pull request until + # the matrix grows. typescript-mutations: needs: generate if: inputs.mode != 'replay' @@ -363,11 +370,13 @@ jobs: retention-days: 14 # The Go lane bounds the whole workflow: every mutant replays the generated - # cohort, about 1.6 minutes per mutant on a fast runner and about 3 on the - # slow class (13 mutants took 25 minutes in run 34660598461 and 47-48 in - # runs 34669546872 and 34670045249), strictly in sequence. A shard compiles, - # runs the baselines (2-4 minutes) and its slice of the catalog; one hung - # cohort adds its 8-minute test bound before the shard fails. + # cohort, about 2.5 minutes per mutant on a fast runner and about 4.5 on the + # slow class (run 40146831443: 145 s per mutant on shard 3, 270 s on shard + # 2, whose seven mutants took 37 minutes), strictly in sequence. A shard + # compiles and runs the baselines (2.7 to 5.3 minutes from job start in the + # 35 shards of runs 40142189738, 40146831443 and 40245022083; the pin allows + # 5.5), then its slice of the catalog; one hung cohort adds its 8-minute test + # bound before the shard fails. # typescript/test/formal-validation.test.ts pins the mutants a shard may # hold within the timeout, so catalog growth fails the pull request until # the matrix grows. @@ -378,9 +387,9 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] env: - MUTATION_SHARD: ${{ matrix.shard }}/10 + MUTATION_SHARD: ${{ matrix.shard }}/15 timeout-minutes: 40 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 @@ -433,7 +442,10 @@ jobs: # The Rust lane compiles the crate and its test binaries in release mode # once per mutant (dependencies are cached across mutants) and replays the - # generated cohort in about a minute; the budget mirrors go-mutations. + # generated cohort in about a minute; the budget mirrors go-mutations. On + # the slow class a shard's baselines take about 5.5 minutes and a mutant + # about 3.7 (run 40146831443), so four shards hold seven mutants in about + # 31 minutes. rust-mutations: needs: generate if: inputs.mode != 'replay' @@ -441,9 +453,9 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2, 3, 4] env: - MUTATION_SHARD: ${{ matrix.shard }}/3 + MUTATION_SHARD: ${{ matrix.shard }}/4 timeout-minutes: 45 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 diff --git a/AGENTS.md b/AGENTS.md index fc37ebd4..7ecbcc25 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,8 +47,8 @@ Classify before editing, because the type sets the design record, the steps and - **Mutations:** Replay plus the TypeScript, Go and Rust mutation campaigns (`-f mode=mutations`), which plant catalogued bugs in a port and require its tests to catch them; about 50 minutes. - **Full:** Mutations plus the Quint model checks and the model-fault campaign (`-f mode=full`), - which plants bugs in the models and requires an invariant to catch each one; about two and a - half hours. The campaign reads only the models, the challenge catalog and the Quint toolchain, + which plants bugs in the models and requires an invariant to catch each one; about three + hours. The campaign reads only the models, the challenge catalog and the Quint toolchain, so a change confined to the ports or their replay drivers never needs it. `-f exploration=true` adds a fresh-seed generation and replay in an isolated copy of the source; a move needs it, and the weekly run on `main` includes it. diff --git a/DEVELOPMENT_PROCESS.md b/DEVELOPMENT_PROCESS.md index 66021086..7644313c 100644 --- a/DEVELOPMENT_PROCESS.md +++ b/DEVELOPMENT_PROCESS.md @@ -96,7 +96,7 @@ type needs once, on the final candidate. port and require its tests to catch them, in about 50 minutes. A change that edits a file a catalogued mutant rewrites, or a mutant catalog itself, needs it. - **Full** adds the Quint model checks and the model-fault campaign, which plants bugs in the - models themselves and requires an invariant to catch each one, in about two and a half hours. + models themselves and requires an invariant to catch each one, in about three hours. The campaign reads only the models, the challenge catalog and the Quint toolchain, so a change confined to the ports or their replay drivers never needs it. - **Full with exploration** also generates histories from a fresh random seed in an isolated copy diff --git a/design/2026-10-24-shadow-mismatch-paths.md b/design/2026-10-24-shadow-mismatch-paths.md new file mode 100644 index 00000000..38570168 --- /dev/null +++ b/design/2026-10-24-shadow-mismatch-paths.md @@ -0,0 +1,1000 @@ +# Shadow mismatch paths: name the differing fields in confirmed-mismatch warnings + +- **Status:** Implemented +- **Issue:** #243, which tracked the work after the TypeScript reference; #242 completes it +- **Pull requests:** #242 +- **Baseline:** `main` at `b4e4fba`, the commit the record was written against +- **Supersedes:** the design in pull request #138, which is unmerged; the + decisions it carries are listed under [Carried decisions](#carried-decisions) + +## Context + +Shadow validation compares a sampled cached value with the source and reports +`match`, `mismatch` or `superseded` as a bounded metric. The only way to learn +*what* disagreed is `shadow.logMismatches: true`, which logs the logical cache +key (2 KiB) and the complete native JSON of both values (8 KiB each) after a +confirmed mismatch. That is all or nothing: a team that wants to know which +field keeps disagreeing has to put whole records into logs, and for a large +record the 8 KiB cap usually truncates the dump before the differing field. + +The operational questions a mismatch rate raises are structural: is it always +the same field, is the cache missing a field the source now has, is one list +longer, or is the difference something JSON cannot show at all, such as a +`Date` object meeting the ISO string it was decoded to. None of these need the +values themselves. + +Pull request #138 (August 2026) built most of the answer for TypeScript: a +`mismatchLogging` content group in place of the boolean, a value projector +hook, and a built-in differ over the native-JSON renderings of both sides +emitting `{ type, path, value, oldValue }` entries. The maintainer reviewed it +three times and ended with no remaining findings, but it never merged. It is +TypeScript-only, it predates the Go, Rust and Python ports, the `typescript/` +layout move and this record process, and its branch can no longer be rebased. +Its reviewed decisions are kept; its value-bearing diff is reshaped around a +paths-only tier it lacked. + +Constraints: + +- Logging stays default-off, runs only after a confirmed `mismatch`, inside + detached shadow work, with bounded CPU; the caller's result is never + affected (C60, C50–C54). +- Every port mirrors the config leaves, the warning fields and the hook, and + the ports' value domains differ (B03): Node deep equality, Go reflection over + a portable value domain, Rust `PartialEq` on `T`, Python dict/list. +- Runtime policy merges leaf-wise and `DialCacheKeyConfig.disabled()` is a + complete kill switch, so every new leaf is a scalar the overlay can set. +- Warning fields reach log pipelines, so a path list is a structured array of + strings, not a rendered blob. +- `shadow.logMismatches: true` is live in at least one consumer's dynamic + config store, so the upgrade must not silently change whether a mismatch is + logged or force a deploy-then-config ordering. +- The replay protocol records the warning as an observation whose extra fields + must be JSON primitives + ([PORTING.md](../formal/guides/PORTING.md#trace-and-observation-contract)). + +## Proposal + +The `shadow` group gains a `mismatchLogging` content group with two +independent default-off leaves: `value`, a boolean, and `diff`, an ordered +level `"off" | "paths" | "values"`. `logMismatches` stays as the one-flag +shorthand for the maximal setting: + +```ts +shadow: { + ramp: 5, + mismatchLogging: { diff: "paths", value: false }, +} +// Equivalent to mismatchLogging: { diff: "values", value: true }: +shadow: { ramp: 5, logMismatches: true } +``` + +`logMismatches` is expansion, not a third leaf: it sets a baseline for the +group, and the leaves beside it are the exceptions. Where it appears, in a +default config or a runtime overlay, `true` expands to +`{ value: true, diff: "values" }` and `false` to `{ value: false, diff: "off" }`, +then any leaf written in the same object overrides its expanded value, and +only then does the ordinary leaf-wise merge run, runtime over defaults. The +rule has no special case for `false`: + +| Written | Effective | +| --- | --- | +| `{ logMismatches: true, mismatchLogging: { value: false } }` | `{ value: false, diff: "values" }` | +| `{ logMismatches: false, mismatchLogging: { diff: "paths" } }` | `{ value: false, diff: "paths" }` | +| default `mismatchLogging: { diff: "paths" }`, runtime `logMismatches: false` | both off: the overlay's expanded leaves win by layer order | +| default `mismatchLogging: { value: false }`, runtime `logMismatches: true` | `{ value: true, diff: "values" }`: the runtime baseline overrides the default leaf | + +A dominant `false`, a kill switch that beats leaves written beside it, was +not chosen: it would be asymmetric with `true`, it would make the second row +a silently dead config, and the kill switches exist already, +`DialCacheKeyConfig.disabled()` and a runtime overlay carrying +`logMismatches: false` with no leaves beside it. An invalid value expands to +both leaves off with one `config_resolution`, as today. Everything `diff` +logs derives from the renderings that `value` logs (D3), so the shorthand +logs nothing outside them: a member that `toJSON` drops stays dropped. It +can log more of them, because the walk reads each whole rendering while +`cachedValueJson` and `sourceValueJson` stop at 8 KiB, so for a larger value +`diff` shows differing fields past that cut. + +A warning is emitted after a confirmed mismatch when `value` is true or +`diff` is above `"off"`. It always carries `cacheNamespace`, `useCase`, +`keyType`, `outcome: "mismatch"`, `cachedValueAgeSeconds` (the age the +`observeShadowValueAge` metric records), `cacheKey` (the logical URN, 2 KiB) +and, when warning construction failed, `loggingOutcome` (see Failure +handling). Each setting adds: + +| Setting | Fields | Bound | +| --- | --- | --- | +| `diff: "paths"` | `mismatchPaths: string[]`, RFC 6901 JSON Pointers into the compared value; `mismatchPathCount`, the total before the cap | 64 entries, each pointer 512 bytes | +| `diff: "values"` | The `"paths"` fields, plus `diffJson`: the same entries with their `cached` and `source` values, each side clamped to 1 KiB; a side absent from the entry is omitted | 8 KiB | +| `value: true` | `cachedValueJson`, `sourceValueJson`: native JSON of each side | 8 KiB each | + +The two `diff` levels are projections of one list of entries +`{ path, cached?, source? }`, computed once by the built-in walk or returned +by the hook, so they cannot disagree: `"paths"` logs the paths, `"values"` +adds the entries with their values. The levels are an exposure ladder: +`"paths"` is what a cache of user data can leave on in production, +`"values"` is what an investigation turns on. + +One per-use-case hook sits beside `shadowComparator` on `cached()` and +`getOrLoad()`: + +```ts +dialcache.cached(fetchUser, { + useCase: "GetUser", + keyType: "user_id", + cacheKey: (id) => id, + // Optional. Replaces the built-in computation; receives the compared + // values and returns the entries. Absent: the built-in JSON differ runs. + shadowMismatchDiff: (cached, source) => [ + { path: "/profile/email", cached: cached.profile.email, source: source.profile.email }, + { path: "/roles" }, + ], +}); +``` + +Each entry names a path; its `cached` and `source` sides are optional, so a +hook that only knows where the values differ returns paths alone, and an +entry for a field present on one side carries one side. The hook follows the +comparator's discipline: synchronous, deterministic, side-effect-free, +non-mutating, bounded. A hook that throws, returns a promise-like, returns a +non-array, or returns a member without a string `path` logs `null` for +`mismatchPaths` and `diffJson` and never falls back to the built-in +computation. A hook value with no JSON rendering makes `diffJson` `null` +while `mismatchPaths` stays, since the paths are still valid and logging +less is the safe direction. Hook values are the use case's own code, like +the comparator: the built-in's guarantee that `diffJson` holds only what the +renderings behind `value` contain applies to the built-in only. + +The built-in computation: + +1. Render each side to native JSON once and parse it. A side with no + rendering (a cycle, a `bigint`) makes `mismatchPaths`, `diffJson` and + that side's value field `null`. An absent side, the top-level undefined + marker a negative-cache entry stores, is not a rendering failure (D13). +2. Walk both parsed values together. Objects recurse over the union of their + member names in UTF-16 code-unit order (the order W02 already prescribes + for key arguments); a member present on one side only is reported at its + pointer. Arrays of equal length recurse by index; arrays of different + length are reported once at the array's own pointer without descending. + Any other pair that differs is reported at its pointer; numbers compare + as IEEE doubles, and a boolean never equals a number. Nesting deeper + than 32 levels is reported at the container's pointer without descending. + Each entry carries the two values at its pointer as the walk holds them: + one side when the member exists on one side only, the whole containers + for an array-length or depth entry. +3. Roots of different kinds (object against array, container against + primitive, primitive against primitive) yield one entry at `""` carrying + both roots. An absent root against a present one yields one entry at + `""` carrying only the present side (D13). Byte-identical renderings + yield no entries, so + `mismatchPaths` is `[]`: the difference is outside the logged form, which + means a type the rendering erased or a comparator that sees more than + JSON. +4. Entries are emitted in walk order and the list is cut at 64; each pointer + is clamped to 512 bytes and each value side to 1 KiB with the existing + `...[truncated]` marker; the rendered `diffJson` is clamped at 8 KiB the + way `cachedValueJson` is today; and `mismatchPathCount` carries the uncut + total. + +Failure handling. Everything above runs in detached shadow work after the +verdict, on a job the caller never awaits: the caller already holds its +result, the Redis hit on a served-hit path or its own source result on a +ramped-down path, before the shadow job reads anything (C48, C49). The +verdict and its `shadowValidation` outcome are decided before any warning +work starts and are never changed by it (C60). A throw anywhere in warning +construction is caught and degrades to the metadata-only warning, and a +logger exception is isolated as today. So the rule is fail-open for the +caller, who is never affected, and fail-closed for exposure, since a failure +logs less, never more. Whether the diff computation holds the shadow capacity +slot stays a port detail: TypeScript and Python render after the slot is +released, Go holds it on the shadow job's goroutine until the warning work +finishes, and Rust holds it through the bounded CPU executor. +The caps bound the built-in; a hook is user code under the comparator's +discipline, so a runaway hook costs what a runaway comparator costs. + +Each warning attempt, a confirmed mismatch with logging on, emits one +`shadowMismatchLogging` metric event with the cache labels and an `outcome`: +`hook_error` when the hook or the built-in walk threw or returned an invalid +list, `render_error` when a side had no JSON rendering (an absent side is +not one, D13), `logger_error` when +the logger failed (D15), else `logged`; the outcomes rank in that order, so +`hook_error` wins over `render_error`, which wins over `logger_error`. +Logging off emits nothing, so the metric never shadows the mismatch count. +When the outcome is not `logged`, the warning carries it as +`loggingOutcome`, so a reader sees why fields are null. Invalid logging +config stays `config_resolution`, carried from #138. + +Port shapes: Go `ShadowPolicy.MismatchLogging *MismatchLoggingPolicy` with +`Value *bool`, `Diff *MismatchDiffLevel` (`off`, `paths`, `values`) and +`Operation.MismatchDiff func(cached, +source T) ([]MismatchEntry, error)`, with `MismatchEntry{Path string; +Cached, Source any}` and the port's absence convention for an omitted side; +Rust `ShadowPolicy.mismatch_logging: Option` with +`value: Option` and `diff: Option`, an enum of the three +levels, and an `Operation::mismatch_diff(...)` builder returning +`Vec` with +`Option` sides, where the built-in computation needs +`T: Serialize`, so an operation built with `with_codec` has entries only +through the hook, and the existing `Operation::preview` stays as the +renderer for such types; Python +`shadow={"mismatch_logging": {"diff": "paths"}}` (camelCase accepted as +today) and `shadow_mismatch_diff=` beside `shadow_comparator`, returning a +list of dicts. Runtime config in every port validates each leaf +independently: an invalid leaf acts off (`value: false`, `diff: "off"`), and +one `config_resolution` per call covers every invalid leaf and shorthand; +unknown fields are ignored, as on `main` (D14); a non-object `mismatchLogging` +fails resolution like a malformed layer map. `logMismatches` keeps its type in every port (Go +`LogMismatches *bool`, Rust `log_mismatches: Option`, Python +`log_mismatches` or `logMismatches`) and expands before the leaves merge. + +## Out of scope + +- A metric for paths. Pointers are unbounded labels. Deferred until log + pipeline aggregation proves insufficient; a bounded label derived in + `shadowMismatchDiff` is possible without library support. +- A TypeScript-only leaf type classification that reports `/updatedAt` when a + cached ISO string meets a source `Date`. Deferred until real logs show `[]` + dominating TypeScript mismatches; a use case can set `shadowMismatchDiff` + for it today. +- Array alignment (longest common subsequence or alignment by an identity + field). Rejected for the built-in because it is quadratic in the worst case + and needs a per-use-case identity hint; the hook covers it. +- Aggregating paths inside the library (top paths per window). Pipeline work. +- A value projector hook, #138's `shadowMismatchLogValue`, which redacted + values in code before `value` and `diff` logged them. Deferred: + `diff: "paths"` is the value-free level, `diff: "values"` narrows values to + the differing leaves, and an optional hook is additive later. Reopen for a + use case that needs the differing values logged while the same values carry + data that a path list cannot express; a `toJSON` on the value class or a + wrapping logger covers it meanwhile. +- A textual diff of the two renderings. It reads well and pinpoints a change + inside a long string, but it cannot be aggregated by field, it is a + presentation of two strings `value` already logs, and pinning it across + ports would need byte-identical pretty-printing and a fully specified + line-diff algorithm. A later refinement of `diff` entries can trim the + common prefix and suffix of long string leaves instead. +- A key projector. Deferred as in #138. +- Failing closed when shadow policy is enabled without a shadow observer + (#151). Independent. + +## Contract and model changes + +- **C60** changes its wording: opting in means `value` is true or `diff` is + above `"off"`. With the last port it also gains the `shadowMismatchLogging` + outcome rule (D11, D12): one outcome per warning attempt, nothing with + logging off, and `hook_error` over `render_error` over `logger_error`. Its + native clause stays: JSON conversion of native values, exact truncation text + and the logger backend remain B03/X01. +- **W10**, a new obligation: the built-in computation in the Proposal, as a + portable function over JSON values, with the array rule, the root cases, + the ordering and the caps. + W10's evidence is a hand-authored `mismatchPathVectors` group in + [protocol-vectors.json](../formal/catalogs/protocol-vectors.json), each + vector a pair of JSON values with the expected entries, paths and values + compared as parsed JSON so each port's string rendering stays a B03 + detail, mapped in + `semantic-cases.json` as `protocol/mismatchPathVectors/*` and replayed by + every port's protocol-vector test. The walk over nested values has no + relation to kernel state, so Quint models only its scalar cases, the ones + the shadow profile's replay reaches (D12): `[]` for identical renderings, + else one entry at `""`. The completion inventory requires + every port to replay every vector, so the group and the row land with the + last port (Plan step 4); until then the TypeScript differ is pinned by + native tests. +- **Quint (D12):** `kernel/shadow.qnt` replaces the `logging` flag of + `Shadowing` and `Job` with the captured `{ value, diff }`, and keeps one + `Malformed.logging` flag, since one `config_resolution` covers any number of + invalid inputs; the shorthand expansion, the leaf-wise merge and per-leaf + resolution are pure functions there. The dark arm's channel row + (`DiagnosedShadowed` in `kernel/shadow.qnt`, with + `Diagnostics::ShadowChannel`; not the base `Diagnostics::Channel`) gains a + warning-content list and a logging-outcome list. The shadow profile's + `ShadowDiagnostics` carries them; the dark-layers and shadow-read-deadlines + profiles' channel types and the `held-dark-shadow` fixture carry them empty, + without a version change, and the shared `diagnostics` parser in + `features.mjs` accepts the two lists as absent or empty for a profile that + does not assert them, so `make differential` still compares those profiles + against reference traces that lack them. + `dialcache-shadow-conformance.qnt` gains starting and runtime choices + for each `diff` level, `value` and the shorthand. It states each captured + leaf per leaf (`capturedLoggingFollowsLayers`), because an invalid leaf + beside a valid one leaves the valid leaf on: a leaf whose effective value + came from an invalid input is off. That invariant implies `main`'s + `invalidLoggingCannotBeCaptured`, which is deleted. It adds invariants + (one outcome per warning attempt, none with logging off, field groups + matching the captured leaves) and regressions for the four-row table and + for each failure outcome alone and combined. Served-hit jobs reach the same + warning code as dark jobs in every port, so the profile's dark jobs stand + for both; a hook's successful entries stay pinned by native tests. The + shadow profile version goes from 3 to 4, so `make differential` reports it + as an intended divergence; unchanged + behavior is shown by the existing regressions, which keep their warning + counts, and by the profiles that keep their versions. Until the last port + lands, the `logPolicy` choices in `formal/replay/features.mjs` and the five + fixed scenarios in `behavioral-scenarios.json` keep `logMismatches`, which + stays valid, so the shared corpus needs no change while the ports catch up; + the five fixed scenarios stay after that. +- **Observation record:** the `mismatchWarning` event in + [PORTING.md](../formal/guides/PORTING.md#trace-and-observation-contract) + gains optional `cachedValueAgeSeconds`, `mismatchPathsJson` (the array as one + JSON string, keeping fields primitive), `mismatchPathCount`, `diffJson` and + `loggingOutcome`, and a `shadowMismatchLogging` event kind with the cache + labels and `outcome` joins the table, predicted by the shadow profile (D12). + Every port's driver records them and gains the profile's three failure + inputs, and all four `formal-` lanes run. Every port compares warning + events exactly against shared expectations, so until every port emits the + new fields the TypeScript driver records only the fields all ports share. +- **Native (B03/X01):** hook execution per port (thenables in TypeScript, + panics in Go, the bounded CPU executor in Rust, the event loop in Python); + which native types each renderer sees; the logger backend; the exporter + names and schema of the `shadowMismatchLogging` metric; the native value + each driver uses for the unrenderable source input and for an absent + value, and how its hook and logger + fail (throw, panic or returned error). + +## Decisions + +### D0. Carry the decisions reviewed on #138 unchanged + +- **Choice:** the [Carried decisions](#carried-decisions) below are taken as + settled; this record does not reopen them. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." +- **Rejected:** + - Re-deciding each one here. They were reviewed and approved in #138's + threads; relitigating costs a round trip and invites drift. + +### D1. Which content leaves exist + +- **Choice:** `value`, a boolean, and `diff`, an ordered level + `"off" | "paths" | "values"`; `cacheKey` is part of every warning. #138's + `key` leaf is dropped. The cost accepted: a string leaf among boolean and + numeric ones, so each port carries a three-value type with its validation. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." +- **Rejected:** + - `key`, `value`, `diff` as in #138. The `key` leaf existed only so a + warning could name no entity; a cache key is an identifier the + application already sends to Redis, and `main` has always logged it. One + fewer leaf to merge, kill and mirror. + - Three booleans, `paths`, `value` and `diff`, this record's first shape. + Rejected 2026-10-24: `paths` and `diff` are an exposure ladder, which an + ordered level states directly, and `{ paths: false, diff: true }` was a + state with no purpose. + - A single level for the whole group (`off | paths | values | everything`). + `value` and the diff level are independent choices: an investigation may + want the whole values with or without the structural list. + - `value` and a boolean `diff` only, logging the paths inside `diffJson`. + The paths-only mode is the one a cache of user data can leave on in + production; without it, value-free logging moves back into code. + - A `diff` level without `"values"`, so `off | paths` beside `value`. The + whole-value dump is truncated at 8 KiB, so for a large record the + `"values"` level is the only way to see the differing leaves without + raising the cap. + +### D2. Path encoding + +- **Choice:** RFC 6901 JSON Pointer strings (`/profile/email`, `/items/2`, + `""` for the root), `~0` and `~1` escaping. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." +- **Rejected:** + - Segment arrays `["profile", "email"]` as in #138. Unambiguous but verbose, + and log pipelines cannot group by them without flattening. + - Dotted paths `profile.email`, `items[2]`. Ambiguous when a key contains + `.` or `[`. + - JSONPath. Nothing here needs a query language. + +### D3. What the built-in computation compares + +- **Choice:** the parsed native-JSON renderings of both sides, rendered once + each, with no entries for byte-identical renderings and one entry at `""` + for roots of different kinds. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." +- **Rejected:** + - The native values the comparator saw. It would explain every verdict, + including a `Date` against its ISO string, but each port's value domain + differs (B03), so there is no portable rule or vector, and the diff could + expose fields that `toJSON` removes from `value` logging, the leak #138's + review closed. + - Making the differ the default comparator, with early exit for the + verdict. The default verdict would change in every port (C50), and a + custom comparator still breaks the unification. + - A root sentinel when renderings are identical. An empty `mismatchPaths` + already says it, and the root entry is kept for a difference the walk + cannot descend into. + +### D4. Arrays + +- **Choice:** equal lengths recurse by index; different lengths report the + array's pointer once and do not descend. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." +- **Rejected:** + - Index-wise always, as in #138. One inserted element reports every later + index; for a 50-element list that is 50 misleading pointers. + - Wildcard collapse (`/items/*` once more than K indices differ). Keeps + precision for small changes, but K is a magic number and fewer than K + shifted elements still report as changed. + - Alignment. See Out of scope. + +### D5. Walk order + +- **Choice:** object members in UTF-16 code-unit order, arrays by index, + entries emitted in walk order. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." +- **Rejected:** + - Insertion order. Go map iteration is randomized and Rust's `serde_json` + map is ordered by key, so insertion order cannot be matched across ports. + - Code-point or byte order. W02 already sorts key arguments by UTF-16 code + units, so every port has that comparison; a second ordering is a second + thing to get wrong. + +### D6. The custom hook + +- **Choice:** `shadowMismatchDiff(cached, source)` receives the compared + values and returns entries `{ path, cached?, source? }`; absent means the + built-in runs; a throw, a promise-like, a non-array or a member without a + string `path` logs `null` for both `mismatchPaths` and `diffJson` and never + falls back to the built-in; a value with no JSON rendering nulls `diffJson` + alone. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." +- **Rejected:** + - `shadowMismatchPaths(cached, source) => string[]`, with the library + resolving each pointer against the JSON renderings to produce `diff`. + Rejected 2026-10-24: resolution meant unescaping, index parsing and + unresolvable pointers in four ports, and could never produce values for + a type without a JSON rendering; values that ride along with the walk or + the hook need none of it. + - #138's `shadowMismatchLogDiff` returning arbitrary JSON. A required + `path` and optional sides keep the output bounded and let `mismatchPaths` + derive from the same list. + - Falling back to the built-in when the hook fails. A broken hook would + then log more than its author intended. + - Dropping invalid members instead of nulling the result. Partial output + from a misbehaving hook is a guess. + +### D7. What each `diff` level logs + +- **Choice:** two projections of one entry list. `mismatchPaths` is the + entries' paths as strings; `diffJson` is the entries with their values as + one rendered string capped at 8 KiB; `diff: "paths"` logs the first and + `diff: "values"` both. The built-in attaches both values to each entry as + the walk emits it; the hook supplies its own. The leaves are independent: + `{ diff: "values", value: false }` logs where and what without the + whole-value dumps. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." +- **Rejected:** + - Dropping `diff` from the first version as too complex. Considered + 2026-10-24: the complexity was pointer resolution, which this shape + removes, and the truncation case, values past the 8 KiB `value` cap, is + what `diff` exists for. + - A separate diff hook beside a paths hook. One list decides where and + what, so the two fields cannot disagree. + - A structured array in the log. A byte cap cannot be stated for it, and + the observation record needs primitives. + +### D8. Portable evidence for the built-in computation + +- **Choice:** a hand-authored `mismatchPathVectors` group in + `formal/catalogs/protocol-vectors.json`, replayed by every port. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." +- **Rejected:** + - Quint-generated vectors. Quint has no recursive types for nested JSON, + and the function has no relation to the kernel's state machine. + - A generator that computes expected outputs with the TypeScript + implementation. The fixture would never catch a TypeScript bug. + - Native tests only, as a B03 detail. Four hand-written differs would drift + in exactly the cases vectors pin: escaping, ordering, the array rule, the + root cases. + +### D9. Caps + +- **Choice:** 64 entries, 512 bytes per pointer, 1 KiB per value side, depth + 32, `diffJson` 8 KiB, `mismatchPathCount` carries the uncut total; a cut + list is visible because the count exceeds the length. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." +- **Rejected:** + - A byte cap on the path list. The list is structured, and a count is what + a pipeline can test. + - No depth cap. The walk runs on parsed JSON, so it terminates, but a + pathological depth is CPU on the detached budget for no diagnostic value. + +### D10. What happens to `shadow.logMismatches` + +- **Choice:** it stays, permanently, as the one-flag shorthand for the + maximal setting, `{ value: true, diff: "values" }`, with the expansion + semantics in the Proposal. The convention + against aliases and shims covers unused API; this field is live in a + consumer's config store, and the shorthand is a coarse control with a + defined meaning, not a deprecated path. A record that adds a leaf decides + whether the shorthand covers it. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." The exposure wording in the Proposal and in the rejected + `{ value: true }` expansion below was corrected 2026-10-24, after review + showed that `diff` reads past the 8 KiB value cut; the maintainer kept the + choice, in session: "1 sounds reasonable." +- **Rejected:** + - Removing it and treating a stale value as an ignored unknown field with a + `config_unknown_field` metric, as #138 decided on 2026-08-15. Rejected + 2026-10-24: the upgrade would silently stop mismatch logging for every + config store that carries `logMismatches: true`, and the migration would + have to be sequenced after the deploy. + - Keeping it temporarily, with an issue for its removal. The removal is a + second breaking change plus the same config flip, with the same + silent-loss risk at that moment; the cost is deferred, not saved. + - Expanding `true` to `{ value: true }` only, preserving today's output + exactly. All-on logs nothing outside the renderings `value` logs, and it + delivers paths to existing configs without a change. For a value over + 8 KiB it also shows differing fields past the dump's cut, which a team + that has opted into `logMismatches: true` already accepts for smaller + values, since truncation is not redaction. + - A third leaf merged independently and OR'd at read time. An explicit + `value: false` could then never win over an inherited `true`. + - A `false` that beats leaves written beside it. Asymmetric with `true`, + and `{ logMismatches: false, mismatchLogging: { diff: "paths" } }` would + be a silently dead config; `disabled()` and a bare `logMismatches: false` + overlay are the kill switches. + +### D11. How warning-pipeline failures are observed + +- **Choice:** a dedicated `shadowMismatchLogging` metric with the cache + labels and `outcome` in `logged | hook_error | render_error | logger_error`, + one event per warning attempt, nothing when logging is off, and + `loggingOutcome` on the warning when it is not `logged`. +- **Status:** Confirmed 2026-10-24, in session: "I confirm D0 through D11 as + proposed." D15 widens `logger_error` from a throw to any observable logger + failure, an asynchronous one included. +- **Rejected:** + - Nothing, as today: a null field in a log line. A hook that has thrown + for a week is invisible, and a logger that throws is silent. + - A new category on the `error` counter. That counter's `in_fallback` + semantics describe cache plumbing; a verdict-style outcome metric + mirrors how `shadowValidation` and `staleRecovery` already report. + - Changing the `shadowValidation` outcome. C60 keeps the warning + independent of the outcome metric, and the verdict is decided before any + warning work starts. + - Truncation as an outcome. It is not a failure and co-occurs with + `logged`; `mismatchPathCount` against the list length and the + `...[truncated]` marker already show it. + +### D12. What the shadow profile predicts about mismatch warnings + +- **Choice:** the `shadow` profile predicts what each confirmed-mismatch + warning contains, not only that one is emitted. A shadow job captures + `value` and `diff` at admission in place of the one `logging` flag; + `Malformed` keeps one flag, since one `config_resolution` covers any number + of invalid inputs. For each warning a dark job emits, the model predicts + which of the Proposal's fields it carries, which are `null`, its + `mismatchPaths` and `diffJson` entries, its `cachedValueAgeSeconds`, and its + `shadowMismatchLogging` outcome. Every port's replay compares its warning and + event with that prediction, the paths and entries as parsed JSON. The + model's values stay scalars, so the predicted entries are `[]`, one entry at + `""`, or `null` after a failure; the walk stays with the vector group, and D8 + stands. Served-hit and dark jobs reach the same warning code in every port, + so the profile's dark jobs stand for both. + - **Inputs.** Three inputs make each failure outcome reachable, alone and + together: a `shadowMismatchDiff` fixture that throws (distinct from the + existing `shadowHook` fixture, which is the outcome hook), a source + result that fails to render in every port (D13), and a logger-only + failure. + - **Shared tests.** Besides the generated shadow corpus and the five fixed + scenarios, every port runs the same tests with the same inputs and the + same expected outputs, in two groups: the `mismatchPathVectors` payload + pairs with their entries (D8), which check the walk, and named Quint + regression runs, one per row of the Proposal's four-row table and per + failure outcome alone and combined, which check the warning's fields, its + `null` fields and its `shadowMismatchLogging` outcome. Each port's native + tests keep what is native (rendering text, hook execution, the logger + backend and exporter names) and a hook's successful entries, which no + shared input reaches. + - **Rust.** The Rust driver's value type gains `Serialize`, with an + unrenderable value that fails to serialize and an absent value the + warning treats as absent (D13), and the driver builds + its operation so the built-in walk runs; the Proposal's Rust shape is + unchanged. + - **What this buys.** The field, `null` and outcome rules are checked in all + four ports across every policy change, confirmation and failure the + generator reaches. + - **The cost accepted.** The shadow profile version goes from 3 to 4, + every driver gains the new inputs, and the generated corpus needs more + samples to reach every combination. + + This lands with the last port (Plan step 4). +- **Status:** Confirmed 2026-10-24, in session: "confirm". +- **Rejected:** + - Keeping one logging flag, as the record was accepted with: `logPolicy` + choices for each `diff` level and `value` as a driver fixture change, with + warning content pinned only in shared fixed scenarios. Rejected 2026-10-24: + the walk is a pure function of two values, which vectors cover (D8), but + warning content depends on policy history: the leaves captured at + admission, a runtime change before the verdict, an invalid leaf beside a + valid one, a failure at each stage. Generated histories would reach those + combinations with nothing predicting their content, and fixed scenarios + check only the combinations someone writes down. + - Structured values in the model, so its predictions exercise the walk. + Quint has no recursive types, so the walk would be written for one fixed + shape or tabulated per pair, which duplicates the vector group; every + driver would have to seed and load structured values; and default + comparators differ per port on structured values (B03). + - Predicting rendered strings. Pinning text would make rendering a portable + obligation, which C60's native clause leaves to each port (B03); Quint has + no string operations either, so each rendering would be a lookup table. + - Predicting content without the failure inputs. Only `logged` would be + reachable, so the precedence of `hook_error` over `render_error` over + `logger_error` would stay unchecked. + - In Rust, walking the `preview` renderings when `T` is not `Serialize`. It + would change the Proposal's Rust shape, where an operation built with + `with_codec` gets entries only through the hook, to serve a test. + +### D13. An absent compared value is a side, not a rendering failure + +- **Choice:** a port's top-level undefined marker, the value its default + codec stores for a negative-cache entry (TypeScript `undefined`, Go + `Absent`, Python `UNDEFINED`), is an absent side, not a value with no + rendering. When one compared value is absent and the other is present, + the built-in computation reports one entry at `""` carrying only the + present side, as it reports a member present on one side only, so + `mismatchPaths` is `[""]` and `mismatchPathCount` is 1. The absent side's + value field is `null` and the outcome is `logged`: since a failure always + sets `loggingOutcome`, a `null` value field without it means the side was + absent. Two absent values compare equal and never reach a warning, and a + hook still receives the raw values. `render_error` keeps every value that + has no rendering: a cycle, a `bigint`, a throwing `toJSON` and their native + counterparts. Rust's default codec stores no distinct undefined marker, so + the case does not arise for its native values; the Rust driver's value type + gives the model's absent value the same treatment. D12's absent source + input becomes a source result that fails to render in every port: a + throwing `toJSON` in TypeScript, a failing `MarshalJSON` in Go, a failing + `Serialize` in Rust and a non-serializable object in Python. +- **Status:** Confirmed 2026-10-24, in session: "confirm D13". +- **Rejected:** + - Keeping the carried rule that every side with no rendering fails closed + (#138; see Carried decisions) for the undefined marker. A cached + negative-cache entry that meets a real source value, or a loader that + now returns the marker against a stale cached value, is ordinary drift. + Reporting it as `render_error` with `null` paths makes the one outcome + that means "logging broke" fire on healthy data. + - Rendering the marker as JSON `null`. It would blur a stored `null` with + absence, which each default codec keeps apart, and an absent value + against `null` would show no paths though the comparator saw a + difference. + - A separate outcome such as `absent`. It adds a metric label for ordinary + data, and the entry at `""` with one side already says what happened. + +### D14. This change keeps `main`'s config reads + +- **Choice:** apart from the new leaves, config is read as on `main`. Unknown + key-config fields stay ignored and `config_unknown_field` is not added, and + `shadow` and its fields are read as ordinary properties, so an inherited + `shadow` group applies as it does today. The maintainer wants strict validation + instead: a runtime config that fails validation is rejected as a whole, an + error metric is recorded, and the call runs uncached, as one invariant in + every port. That rule replaces both the ignored unknown fields and the + per-leaf fail-open for every config field, so it gets its own design record + and pull request. Until it lands, this change's leaves follow the per-leaf + rules above. +- **Status:** Confirmed 2026-10-24, in session: "no typo checking, we should + just do strcit checking for the config and if it fails it fails and we emit + an error metrics and turn teh cache off. we should maintain this behavior + and invariant in the whole library", with "New record + separate PR" chosen + for where it lands; then "for 242, lets just leave same behavior as in main + for configs, and just remove typo shit, since i nevre asked for it" and, for + the own-property reads, "ok lets do that". +- **Rejected:** + - The carried #138 rule (see Carried decisions): ignore unknown fields and + report each observed config as `config_unknown_field`. A misspelled field + still has no effect, so the report only says that config ran wrong, and + cheap reporting needed a per-config marker on every construction. + - Strict validation inside this pull request. It changes every config field + and every port, well beyond mismatch paths. + - The carried #138 own-property reads of `shadow` and its leaves. They change + how an inherited `shadow` group behaves for configs that exist today, + which mismatch paths do not need. + +### D15. A logger that fails asynchronously is a `logger_error` + +- **Choice:** `logger_error` counts every logger failure the port can + observe: a throw or panic, a returned error, and a returned promise or + awaitable that rejects. When the logger returns a promise, the + `shadowMismatchLogging` event waits for it: rejection gives + `logger_error`, resolution gives `logged`, and a promise that never + settles emits no event. An earlier `hook_error` or `render_error` is + recorded at once, without waiting, since it wins either way. +- **Status:** Confirmed 2026-10-25, in session: "oh ok yea it seems like ti + shoudl be logger_error if we cant log right". +- **Rejected:** + - Counting only a synchronous throw, as D11 first worded it. An + asynchronous transport that drops every warning would report `logged`, + which is the silent failure the metric exists to show. + - Bounding the wait with a timeout. It adds a timer per warning and a + fifth outcome for a logger that is merely slow. + +## Carried decisions + +Reviewed and approved by the maintainer on #138; carried unchanged: + +- Runtime reads of the whole `shadow` group, and of each leaf, are + own-property reads at every boundary. D14 drops this: config reads stay as + on `main`. +- The differ traverses own members of parsed JSON only; no hostile-intrinsics + hardening (the maintainer withdrew that finding). +- Each side is rendered to JSON exactly once; every logged field derives from + that snapshot, so a stateful `toJSON` cannot put into one field what + another omits. +- A side with no rendering fails every field it feeds closed to `null`; no + `null → null` entries. D13 narrows this: an absent side is not a side with + no rendering. +- Unknown `mismatchLogging` leaves and other unknown key-config fields are + ignored and reported once per observed config as `config_unknown_field`. + (#138 applied this to `shadow.logMismatches` itself; D10 keeps that field + instead.) D14 drops the reporting: unknown fields stay ignored, as on + `main`, until strict runtime validation replaces both. A known leaf with an invalid value acts off with one + `config_resolution`; a non-object group fails resolution like a malformed + layer map. +- `value: true` logs raw JSON. +- `DialCacheKeyConfig.disabled()` sets `value: false` and `diff: "off"`. + +## Impact + +- **TypeScript:** `config.ts`, `internal/runtime-config.ts`, + `internal/shadow-log-json.ts`, `dialcache.ts` (the hook, warning assembly, + `resolveShadowLogging`), `metrics.ts` (the `shadowMismatchLogging` hook and + its outcome type), `datadog.ts` and `prometheus.ts`, `index.ts` exports for + the new config, hook and metric types, `scripts/test-package.mjs`, the + shadow, config and exporter tests, the formal behavior driver. +- **Go:** `policy.go` (leaves, shorthand expansion, merge and resolution), + `shadow.go` (warning assembly), the new `shadow_log.go` (walk, caps, hook and + rendering), `metrics.go` (`warnDelivered`, the `shadowMismatchLogging` event + kind and its exporter mapping), `api.go` (`Operation.MismatchDiff`, + `MismatchEntry`), `metrics_datadog.go`, `metrics_prometheus.go`, + `*_driver_test.go`, `protocol_test.go`, `internal/testsuite/policy_test.go`. +- **Rust:** `policy.rs`, `shadow.rs`, the new `shadow_log.rs` (rendering and + warning content) and `protocol/mismatch.rs` (the walk), `observe.rs` + (`ShadowMismatchDetails`, `Display`, the new `Event` variant), `metrics.rs`, + `datadog.rs` and `prometheus.rs` exporters, `operation.rs` and + `use_case.rs` (the hook), `preview.rs`, `testing.rs` (the test-only absent + value), `tests/formal/driver.rs`, the protocol-vector and exporter tests. +- **Python:** `config.py`, `cache.py`, the new `_shadow_log.py`, + `_callbacks.py`, `tests/formal/driver.py`, `tests/test_protocol_vectors.py`, + `test_config.py`, `test_shadow_log.py`. +- **Formal:** `CONTRACTS.md` (C60 wording, the W10 row, the page and test + inventories), `SPEC.md` (mismatch logging rows), `PORTING.md` (observation + row), `GO-PARITY.md`, `behavioral-scenarios.json` (five fixtures), + `features.mjs` (fixture and `logPolicy` encoding), `semantic-cases.json` + (vector mapping and new cases), `protocol-vectors.json` (new group), + `mutations.json` and `rust-mutations.json` (M08 and M09 anchors move to the + new leaf code; new mutants for the array rule, the root cases, the + shorthand expansion and hook fail-closed behavior), the ledgers + `source-audit.json` and `go-parity.json`, and any course lesson whose + citation moves. Under D12 also `kernel/shadow.qnt`, + `kernel/diagnostics.qnt`, `dialcache-shadow-conformance.qnt` and the other + models that spell a `Shadowing` record (the admission, dark-layers, + shadow-layers and shadow-read-deadlines profiles and the `held-shadow`, + `held-dark-shadow` and `shadow-jobs` fixtures), `features.d.mts`, + `witnesses/shadow.mjs` and `shadow-diagnostics.mjs`, `divergence.mjs`, + `coverage-witnesses.json`, `witness-baseline.json`, `execution.json`, + `profiles.json` and `BEHAVIOR.md`. +- **Docs:** `shadow-validation.md` (Confirmed mismatch logging, shared by + every port with native notes per language), `observability.md` (Confirmed + mismatch warnings, the shadow outcomes section and the Datadog and + Prometheus metric tables), `configuration.md`, `api.md`, the language + guides, the port READMEs that name `logMismatches`, and `upgrading.md`: + `shadow.logMismatches` stays and now also logs `mismatchPaths`, + `mismatchPathCount` and `diffJson`; no config migration is required, and + the Go and Rust entries list the code changes. +- **Release:** additive for TypeScript and Python; Python's + `Policy.disabled()` and a merged runtime policy now carry the expanded + `mismatch_logging` leaves in place of `log_mismatches`. Go and Rust + replace the old shapes and delete what becomes unused, at the maintainer's + direction in session on 2026-10-25 ("Replace and delete"). Go adds + `ShadowPolicy.MismatchLogging` and replaces `ResolvedShadow.LogMismatches` + with `LogValue` and `LogDiff`, so unkeyed `ShadowPolicy` and + `ResolvedShadow` literals stop compiling, and it removes + `ShadowMismatchLogDetails`, `ShadowMismatchDetails` and + `PreviewShadowLogJSON`. Rust's `ShadowPolicy` gains a public field, so + existing struct literals stop compiling; `Operation` gains a private field + and can no longer be built with a struct literal; `ShadowMismatchDetails` + and `ResolvedShadow` change shape; and `Event` and `MetricKind` gain + variants that exhaustive matches must handle. The pull request still + carries neither `!` nor a `BREAKING CHANGE:` footer, at the maintainer's + request in session on 2026-10-24: "yes lets drop it and not use `!` in + description since i dont want to increment major version right now". The + Go and Rust entries in `docs/upgrading.md` name the changes. + `shadow_mismatch_logging.count` becomes the longest metric suffix of the + Datadog adapters that TypeScript, Go and Rust bundle, so their longest + valid namespace drops from 175 to 170 characters; the maintainer kept the + name in session on 2026-10-24: "lets keep the shadow_mismatch_logging.count + and just reduce documented max size ofr namespace". `docs/upgrading.md` and + `docs/observability.md` state it. + +## Plan + +1. #242 opens with this record as its first commit and stays at the design + gate until every decision is confirmed and the record is `Accepted`. D12, + added after acceptance, returned the record to `Proposed` until the + maintainer confirmed it the same day; D13, added during the TypeScript + review, did the same until he confirmed it the same day. +2. #242 then carries the TypeScript reference: the leaves and the shorthand, + the built-in differ, the hook, the warning fields, the + `shadowMismatchLogging` metric and its exporters, the TypeScript replay + driver, C60's wording, the TypeScript docs and catalogs, and native tests + that pin the differ, with D13's absent side. The + shared corpus keeps `logMismatches`, which stays + valid in every port, so the other ports keep passing. Tier: Replay and + Mutations, since M08 and M09 anchor on the moved logging code. +3. Once the maintainer is satisfied with the TypeScript reference, #242 also + carries Go, Rust and Python against the TypeScript semantics, with their + drivers and docs. This replaced one pull request per port on 2026-10-24, + in session: "we should just implement these ports in same pr once we are + satsified ok with typescript". +4. With the last port, #242 adds the vector group, its `semantic-cases.json` + mapping, the new contract row (W10) and C60's outcome rule; under D12, the + kernel's captured leaves, the shadow profile's starting and runtime + choices for each `diff` level, `value` and the shorthand with their + fixture and `logPolicy` encodings, its warning and outcome predictions, + the three failure inputs in every driver (the unrenderable source result + joins the model's source results, D13) and the Rust driver's + serializable value type; records the new observation fields in every + driver; and catalogues mutants for the built-in computation (the array + rule, the root cases) and for the hook's fail-closed behavior in every + port, plus C60's first model-fault challenge, mapped to new mutants and to + M08, M09 and M65 (M65 gains a Rust edit). Tier: Full (Portable behavior). + +## Deviations + +- **2026-10-24, #242:** the built-in computation's CPU and transient memory + grow with the rendered size of both values and with the number of members + and elements they hold, linearly apart from sorting each object's member + names: the uncut `mismatchPathCount` (D9) needs a full parse and walk, which + holds both parsed trees, and the caps bound only the output. The + Constraints' "with bounded CPU" and the Proposal's "The caps bound the + built-in" do not describe this. Review measured about 86 ms per warning on + a 5.6 MB value in TypeScript, against 13 ms for the value fields alone. On + 2026-10-25, at the merge of the four ports, one Apple-silicon Mac measured + a 6.3 MB object of 150,000 string members at `diff: "paths"`, against the + value fields alone: TypeScript about 0.12 s against 0.06 s and Python about + 0.18 s against 0.04 s, both on the event loop after the shadow slot is + released; Go about 0.6 s against 0.5 s on the shadow job's goroutine + holding the slot, most of it the JSON rendering the value fields already + paid; Rust about 0.09 s against 0.006 s in a release build, on the bounded + CPU executor holding the slot. By default that executor is one + process-wide pool of two worker threads and two queued jobs, which every + cache instance also uses for zstd decompression and for compressing large + payloads, so while warnings occupy it, a compressed hit whose decompression + it rejects falls through to the source; at most `shadow_max_in_flight` + warnings per instance, times the number of instances, contend for it. + `docs/upgrading.md` and `docs/shadow-validation.md` state the cost in every + port. The maintainer kept this behavior as documented in session on + 2026-10-25, and asked for a faster implementation, TypeScript first, as a + follow-up: #248. +- **2026-10-26, #242, TypeScript and Python:** DialCache no longer validates the + hook's entries. D6's rule that a member without a string `path` fails is + dropped, at the maintainer's decision in session that day: "ok lets stop + being so defensive. if a custom differ returns non correct results its fine". + A hook that throws, returns a promise-like or returns a non-array still gives + `hook_error` and never falls back to the built-in computation. Only the first + 64 entries are read, inside the hook's catch, so an entry that throws when + read is still `hook_error`. Their paths are logged in string form: TypeScript + stringifies any value, and Python stringifies any value under a `"path"` key + but fails an entry that has none. Go's and Rust's types already require a + string path. +- **2026-10-26, #242, TypeScript:** the one catch around warning construction + logs every requested field as `null` with `render_error`, not the + metadata-only warning the Failure handling paragraph names, so the warning + keeps the documented rule that each requested field is present. The other + ports keep the metadata-only warning on that path, which no supported input + reaches. The maintainer's rule for the TypeScript trim (2026-10-26): defensive + code only at the config and diff-generator boundaries. +- **2026-10-24, #242:** TypeScript's `cached()` and `getOrLoad()` throw a + `TypeError` for a `shadowMismatchDiff` that is not a function, as they do for + `shouldAttemptStaleRecovery`, rather than letting every warning fail as + `hook_error`; D6 covers only failures while the hook runs. Python's port + follows its existing rule for non-callable options. +- **2026-10-25, #242, Go:** the built-in walk parses each rendering with + `encoding/json`, whose decoder rejects nesting deeper than 10,000 levels, so + a map, slice or `JSONObject` nested that deep renders but fails the walk as + `hook_error`; TypeScript's parser has no fixed limit. Depth is the only + reason a Go rendering does not parse back: a struct renders through + `encoding/json` and is parsed back while it renders, so a struct whose JSON + nests that deep, or whose `MarshalJSON` writes a number outside double + range, has no rendering and reports `render_error` before any walk. A + `diffJson` side clipped at 1 KiB orders object members in UTF-16 order, + since the walk holds parsed Go maps, while `cachedValueJson` keeps a + `JSONObject`'s insertion order; the parsed entries are unaffected. These are + B03 rendering details. +- **2026-10-25, #242, Go:** a Go hook fails by returning an error or + panicking; Go's types rule out a promise, a non-array and a member without a + string path, and a nil slice is an empty list. The metadata-only warning for + a construction failure is kept as a defensive recover, but rendering and the + hook already recover their panics, so no native test reaches it. +- **2026-10-25, #242, Rust:** the built-in walk parses each rendering back + with `serde_json`, and a rendering it cannot parse fails the walk with + `hook_error` where TypeScript compares it: one nested 128 or more levels + deep, which reaches `serde_json`'s parser recursion limit, or, in a build + that enables `serde_json`'s `raw_value` feature without + `arbitrary_precision`, one whose `Serialize` writes a `RawValue` number + outside double range. The vectors nest at most 40 levels. +- **2026-10-25, #242, Rust:** `diffJson` sides from the built-in walk are + parsed `serde_json::Value`s, whose objects render members in UTF-8 byte + order of their names, not the source order TypeScript keeps; a side over + 1 KiB is clamped from that rendering. Entries compare equal as parsed JSON, + so this is a B03 rendering detail. +- **2026-10-25, #242, Rust:** the Proposal's Rust shape keeps + `Operation::preview` as the renderer for operations without `T: Serialize`. + The port keeps `main`'s meanings for it: `Operation::new` and `register` + install a default renderer of the values' JSON, and a `preview` set to + `None` logs no values, as on `main`, at the maintainer's direction on + 2026-10-25. Only the default renderer feeds the built-in walk. A custom + `preview` replaces the JSON for the whole warning, and an operation whose + `preview` is `None` has no JSON rendering, so both, like an operation built + with `with_codec` or registered with `register_custom`, get difference + fields only from a `mismatch_diff` hook. A preview is Rust's redaction path + where TypeScript has `toJSON`, so this keeps D3's rule that `diff` logs + nothing outside the renderings `value` logs. +- **2026-10-25, #242, Rust:** an operation that logs a field it has no + renderer for (a value field without a preview, a difference field without + the default renderer or a hook), and a `preview` that returns `None` or + panics, report `render_error`, so a null field always has a cause (D13). + The maintainer chose `render_error` for the missing renderer, and #245 + tracks a JSON renderer for operations built with `with_codec`. +- **2026-10-25, #242, Rust:** the warning's rendering, walk, preview and hook + run on the bounded CPU executor (see Failure handling). When the executor + rejects that work, or it panics or is dropped, the warning keeps its + metadata, both field groups are absent, and the outcome is `render_error`: + nothing was rendered, and no other outcome describes a warning whose content + never ran. A rejection is not a failure the Proposal's outcomes name. +- **2026-10-25, #242, Rust:** the hook returns `Vec` with + `serde_json::Value` sides, so a panic is its only failure and its sides + always render; `render_error` never comes from hook entries. +- **2026-10-25, #242, Rust:** the absent value of D13 does not arise for + Rust's native values. The replay driver's absent value is the test-only + `dialcache::testing::Absent` (feature `test-util`), which only an + operation's JSON rendering recognizes; a custom `preview` replaces that + rendering, so the preview receives the value. +- **2026-10-25, #242:** step 2 of the built-in computation was edited in + place during review to say that numbers compare as IEEE doubles and that a + boolean never equals a number, and W10 states the same rule. The record as + accepted did not say how numbers compare, and the Python port compared + integers exactly, so it logged paths that TypeScript, Go and Rust did not. + The maintainer confirmed the double rule in session on 2026-10-25: + "Confirm doubles". Two integers that differ only beyond a double's + precision, such as 9007199254740993 and 9007199254740992, therefore log no + path in any port, as a vector pins; exact 64-bit comparison is left for a + later change. +- **2026-10-25, #242, Go:** Failure handling was edited in place during + review to name Go among the ports that hold the shadow capacity slot + through warning work. On `main`, Go released the slot once the job's reads + settled, without waiting for its verdict to be reported, so a slow + `MismatchDiff` hook would have kept a goroutine per confirmed mismatch + outside the slot and a burst of mismatches would have run walks beyond + `WithShadowCapacity`. A job now holds its slot until the verdict is + reported: while the `WithShadowOutcomes` hook, observers and metrics + adapters receive its `shadowValidation` and `shadowAge` events, and while + its mismatch warning is built and logged, the hook and the walk included. + So the Proposal's "a runaway hook costs what a runaway comparator costs" + holds for Go, and a slow hook, observer or logger now shows as `dropped` + jobs beyond capacity; `docs/upgrading.md` states the change. +- **2026-10-25, #242:** Contract and model changes first said the shadow + profile restates `main`'s `invalidLoggingCannotBeCaptured` per leaf, and + the paragraph was corrected in place during review. The per-leaf + restatement was deleted, because `capturedLoggingFollowsLayers`, which + states each captured leaf from its input in force, implies it: both are + checked at `beginCall` over the same jobs, and a leaf whose input in force + is invalid is captured off. The challenge + `shadow-logging-invalid-diff-leaf-acts-values` and the two semantic cases + that cited the deleted invariant point at `capturedLoggingFollowsLayers`. +- **2026-10-25, #242:** the shorthand expansion, the leaf-wise merge and + per-leaf resolution live in a kernel module of their own, + `kernel/mismatch-logging.qnt`, imported only by the shadow profile, rather + than in `kernel/shadow.qnt` as Contract and model changes says, so faults + planted there are not run against the profiles that compose + `kernel/shadow.qnt` with logging off. `kernel/shadow.qnt` keeps the + captured `Logging` leaves, `LOGGING_OFF`, `logs` and the diff levels, and + does not import the new module. diff --git a/docs/api.md b/docs/api.md index c28fd220..cb30fe85 100644 --- a/docs/api.md +++ b/docs/api.md @@ -174,17 +174,21 @@ uncached. | `trackForInvalidation` | `false` | Use watermark-aware Redis reads for this operation | | `fallbackTimeoutMs` | `60_000` | Positive safe integer up to `2_147_483_647` ms; `null` disables the source deadline | | `shadowComparator` | Node strict deep equality | Synchronous, bounded `(cached, source) => boolean`; must not mutate its inputs | +| `shadowMismatchDiff` | Built-in JSON diff | Synchronous, bounded `(cached, source) => readonly ShadowMismatchEntry[]`; replaces the paths and diff of [confirmed-mismatch warnings](shadow-validation.md#confirmed-mismatch-logging) | | `shouldAttemptStaleRecovery` | Instance policy | Synchronous error classifier; replaces the lower-precedence policy | `CacheKeySpec` is a string, number, or bigint id, or `{ id, args? }`. Argument values are string, number, bigint, boolean, `null`, or `undefined`; undefined arguments are omitted. See [Key design](keys.md). -Static defaults, fallback timeout, and stale-recovery classifier are validated -and captured when registering `cached()` or invoking `getOrLoad()`. -The comparator is captured then; its execution and synchronous boolean result -are checked only when shadow comparison runs, with failures reported as -`comparison_error`. +Static defaults, fallback timeout, stale-recovery classifier, and +`shadowMismatchDiff` are validated and captured when registering `cached()` or +invoking `getOrLoad()`; a `shadowMismatchDiff` that is not a function throws a +`TypeError`. The comparator is captured then too. The comparator's +execution and synchronous boolean result are checked only when shadow +comparison runs, with failures reported as `comparison_error`; the diff hook +runs only for a confirmed-mismatch warning that logs a diff, and its failures +leave that warning's paths and diff `null`. Runtime policy is resolved per enabled invocation. These guarantees do not make the entire caller-owned options object deeply immutable; keep definitions stable. @@ -202,16 +206,20 @@ overlay. Omission inherits; it does not turn an inherited field off. | `remoteReadTimeoutMs` | Instance setting, then `50` | Positive safe-integer milliseconds, at most `2_147_483_647`; cannot be unbounded | | `staleOnErrorMaxAgeSec` | Off | Nonnegative safe-integer seconds; `0` disables; positive age must exceed remote TTL and be at most `31_536_000` | | `shadow.ramp` | Off | Independent finite percentage from `0` through `100` | -| `shadow.logMismatches` | `false` | Boolean; controls diagnostic warning output | +| `shadow.logMismatches` | `false` | Boolean shorthand for both leaves below: `true` stands for `{ value: true, diff: "values" }`, `false` for both off | +| `shadow.mismatchLogging.value` | `false` | Boolean; the warning includes the JSON of both compared values | +| `shadow.mismatchLogging.diff` | `"off"` | `"off"`, `"paths"`, or `"values"`; the warning includes the differing fields' pointers, and with `"values"` what each side holds there | Use `CacheLayer.LOCAL` (`"local"`) and `CacheLayer.REMOTE` (`"remote"`) as map -keys. `LayerConfig` is a partial map; `ShadowConfig` describes the shadow group. +keys. `LayerConfig` is a partial map; `ShadowConfig` describes the shadow group, +and `ShadowMismatchLoggingConfig` its `mismatchLogging` leaves, with +`ShadowMismatchDiffLevel` naming the three levels. Tracked Redis physical retention has a separate one-hour cap. | Helper | Result | | --- | --- | | `DialCacheKeyConfig.enabled(ttlSec)` | Sets local and remote TTLs to the supplied value and both ramps to `100`; leaves request-local, shadow, and recovery unselected | -| `DialCacheKeyConfig.disabled()` | Disables request-local and recovery, sets both serving ramps and shadow ramp to `0`, and disables mismatch logging | +| `DialCacheKeyConfig.disabled()` | Disables request-local and recovery, sets both serving ramps and shadow ramp to `0`, and turns both mismatch-logging leaves off | The enabled helper does not create a Redis connection. The disabled helper is an invocation policy, not cancellation or eviction. See @@ -238,10 +246,10 @@ Invalid runtime policy fails open at the affected boundary: | Invalid input | Behavior | | --- | --- | | TTL or serving ramp leaf | Disable that layer with `invalid_ttl` or `invalid_ramp`; record `config_resolution`. Valid layers can continue. Values do not fall back to valid defaults and ramps are not clamped. | -| Config object, layer-map or shadow shape; `requestLocal`, `coalesce`, or `remoteReadTimeoutMs` | Fail resolution for the whole invocation; record `config_resolution` and `config_error`, then run the loader uncached. | +| Config object, layer-map, `shadow` or `shadow.mismatchLogging` shape; `requestLocal`, `coalesce`, or `remoteReadTimeoutMs` | Fail resolution for the whole invocation; record `config_resolution` and `config_error`, then run the loader uncached. | | `staleOnErrorMaxAgeSec` | Disable recovery and record `config_resolution`; a valid ordinary remote layer remains available. A positive age without a remote TTL is also an error. | | `shadow.ramp` | Record remote `config_resolution` and skip shadow work when an eligible Redis path evaluates it; preserve valid serving layers. | -| `shadow.logMismatches` | Disable mismatch logging while preserving shadow work; record remote `config_resolution` only after the metrics hook, cohort, and capacity gates admit the job. | +| `shadow.logMismatches`, `shadow.mismatchLogging.value`, or `shadow.mismatchLogging.diff` | Turn an invalid leaf off; an invalid `logMismatches` expands to both leaves off before the leaves written beside it apply. Shadow work continues. Record one remote `config_resolution` only after the metrics hook, cohort, and capacity gates admit the job. | Validation of layer and shadow leaves depends on traversal: an earlier hit can avoid evaluating lower-layer policy. Unknown runtime fields are generally @@ -296,8 +304,9 @@ See [Coalescing state](coalescing.md#inspecting-process-scoped-flights). | `Serializer` | `dump(value)` returns `string \| Buffer`; `load(payload)` returns `T`; either may return a Promise | | `JsonSerializer` | Default JSON codec, including top-level undefined support; both methods return Promises | -`CachedValue` exposes a function's resolved result type. `ShadowComparator` -and `StaleRecoveryPredicate` name the corresponding synchronous callbacks. +`CachedValue` exposes a function's resolved result type. `ShadowComparator`, +`ShadowMismatchDiff`, and `StaleRecoveryPredicate` name the corresponding +synchronous callbacks, and `ShadowMismatchEntry` is one entry a diff hook returns. See [Direct key construction](#constructing-keys-directly) for defaults, encoding, and validation, and [Serialization](redis.md#serialization) for direct codec behavior, the compile-time guard, and round-trip limitations. diff --git a/docs/configuration.md b/docs/configuration.md index 39fa0fe4..691f3640 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -114,6 +114,12 @@ mappings also accept the shared camelCase names. See the +Within each config, the defaults or a runtime overlay, `shadow.logMismatches` +first expands to the `shadow.mismatchLogging` leaves it stands for, and leaves +written beside it win; only then do the leaves merge, runtime over defaults. The +[mismatch logging table](shadow-validation.md#confirmed-mismatch-logging) +shows the result for each combination. + The policy names in the tables below use the shared JSON configuration shape, also accepted by the Go, Rust and Python policy parsers. Native names and units differ; use the selected language's API reference when constructing typed policy. @@ -160,8 +166,9 @@ the runtime provider to return a sparse per-key ramp override of `0` or `100`. `shadow.ramp` selects its own stable exact-key cohort, independent of both serving ramps. Shadowing additionally needs a valid remote TTL and a metrics -adapter with the outcome hook. `shadow.logMismatches` controls diagnostic -warnings separately and defaults to `false`. +adapter with the outcome hook. `shadow.mismatchLogging` controls diagnostic +warnings separately, choosing the warning's content leaf by leaf, and +`shadow.logMismatches` is shorthand for both leaves, which default to off. [Shadow validation](shadow-validation.md) explains eligibility, comparison, clean-miss fills, capacity, and the data-handling contract. diff --git a/docs/languages/go.md b/docs/languages/go.md index 685e43d2..6c17d74b 100644 --- a/docs/languages/go.md +++ b/docs/languages/go.md @@ -41,6 +41,11 @@ sparse `PolicyOverlay`, shared-shape `JSONPolicy`, or `RawPolicy` on each enable invocation. Nil leaves inherit; pointers such as `Ptr(false)` distinguish an explicit disabling value from omission. +`ShadowPolicy.MismatchLogging` selects confirmed-mismatch warning content with +`Value` and a `MismatchDiffLevel` (`MismatchDiffOff`, `MismatchDiffPaths` or +`MismatchDiffValues`); `LogMismatches` is shorthand for both leaves. See +[confirmed mismatch logging](../shadow-validation.md#confirmed-mismatch-logging). + Durations use `time.Duration`: cache TTLs and recovery ages must be whole seconds, read/source deadlines whole milliseconds. `Operation.SourceTimeout` defaults to 60 seconds when zero; `NoTimeout` explicitly removes that guard. Runtime policy @@ -84,7 +89,9 @@ use this hook only to enable admission rather than forwarding the same outcome twice. See [observability](../observability.md). The default shadow comparator is `SemanticEqual`. An operation's `Comparator` -can implement application equality and return an error. See +can implement application equality and return an error. Its `MismatchDiff` +replaces the built-in difference computation of confirmed-mismatch warnings; a +`MismatchEntry` side set to `Absent` is omitted. See [comparison semantics](../shadow-validation.md#comparison-semantics). ## Conformance diff --git a/docs/languages/python.md b/docs/languages/python.md index a81646a5..c163604a 100644 --- a/docs/languages/python.md +++ b/docs/languages/python.md @@ -110,6 +110,10 @@ automatically schedule synchronous work on threads. The default shadow comparator recursively compares JSON-like values and keeps booleans distinct from numbers. Custom native objects may have different equality semantics; supply `shadow_comparator` for domain-specific equality. +`shadow_mismatch_diff` similarly replaces the built-in mismatch-path +computation of confirmed-mismatch warnings, which the `mismatch_logging` leaves +enable; see +[confirmed mismatch logging](../shadow-validation.md#confirmed-mismatch-logging). Shadow admission requires a metrics observer for terminal outcomes. ## Redis and observability diff --git a/docs/languages/rust.md b/docs/languages/rust.md index fa69cc04..2f597e32 100644 --- a/docs/languages/rust.md +++ b/docs/languages/rust.md @@ -104,6 +104,13 @@ can differ from TypeScript; for example, NaN differs from itself. Supply a domai comparator when appropriate. [Shadow validation](../shadow-validation.md) describes this boundary. +`ShadowPolicy::mismatch_logging` selects confirmed-mismatch warning content. +Its built-in difference computation walks the `serde_json` renderings of an +operation built with `Operation::new` or `register`; a `mismatch_diff` hook +replaces it, and is the only source of differences for an operation whose +`preview` is custom or `None`, or one built with `with_codec`. See +[confirmed mismatch logging](../shadow-validation.md#confirmed-mismatch-logging). + Large compression jobs and all zstd decompression use a bounded CPU executor. Application codecs choose their own scheduling. See [compression](../redis.md#compression) for thresholds and fail-open behavior on executor saturation. diff --git a/docs/languages/typescript.md b/docs/languages/typescript.md index f10c14d4..20c537e6 100644 --- a/docs/languages/typescript.md +++ b/docs/languages/typescript.md @@ -66,6 +66,7 @@ UTF-16 surrogate code units. Successful decoding does not prove schema agreement - [Redis and Valkey](../redis.md): `dialcache/node-redis` and `dialcache/valkey-glide` wrap connected application-owned clients. - [Observability](../observability.md): `dialcache/prometheus` uses a supplied registry; `dialcache/datadog` uses a supplied DogStatsD client. - [Shadow validation](../shadow-validation.md#comparison-semantics): the default comparator is Node's `util.isDeepStrictEqual`; provide `shadowComparator` for domain equality. +- [Confirmed mismatch logging](../shadow-validation.md#confirmed-mismatch-logging): `shadow.mismatchLogging` selects the warning's paths, diff values, and whole values; provide `shadowMismatchDiff` to compute the differing fields yourself. The site and generated API reference follow repository `main`. Use release-tag source and the installed package's declarations for an older release. diff --git a/docs/maintainers.md b/docs/maintainers.md index fce33413..8c52a7f4 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -47,8 +47,8 @@ Complete replay, mutation measurement and exploration run through the manual and weekly formal workflow, using the same Make targets as local runs. Its `mode` input selects `replay` (the generated corpus in every port, about 15 minutes), `mutations` (plus the port mutation campaigns, about 50 minutes) or -`full` (plus the model checks and the model-fault campaign, about two and a -half hours). Before merge, run `replay` for changes to port behavior or a +`full` (plus the model checks and the model-fault campaign, about three +hours). Before merge, run `replay` for changes to port behavior or a replay driver, and `mutations` when the change also edits a file a catalogued mutant rewrites. Run `full` for changes to a model's transitions, invariants or challenges, the kernel library, `formal/tools/` or the Quint toolchain, and diff --git a/docs/observability.md b/docs/observability.md index 674df7fe..784f51ec 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -269,7 +269,9 @@ The Datadog metric namespace must: - produce final metric names no longer than 200 characters. The adapter rejects invalid namespaces and overlong final names instead of -relying on client-side normalization. +relying on client-side normalization. The longest name it builds is +`.shadow_mismatch_logging.count`, so the namespace can be at most 170 +characters. @@ -316,6 +318,7 @@ and bytes without unit conversion: | `dialcache.coalesced.count` | Count | `dialcache_coalesced_counter` | | `dialcache.shadow.count` | Count | `dialcache_shadow_validation_counter` | | `dialcache.shadow.value_age` | Distribution or histogram | `dialcache_shadow_value_age_histogram` | +| `dialcache.shadow_mismatch_logging.count` | Count | `dialcache_shadow_mismatch_logging_counter` | | `dialcache.future_timestamp_offset` | Distribution or histogram | `dialcache_future_timestamp_offset_histogram` | | `dialcache.stale_recovery.count` | Count | `dialcache_stale_recovery_counter` | | `dialcache.stale_recovery.value_age` | Distribution or histogram | `dialcache_stale_recovery_value_age_histogram` | @@ -334,8 +337,8 @@ Labels and meanings are shared with the [metric catalog](#metric-catalog). Synchronous client throws are isolated when DialCache invokes the adapter. DialCache also consumes thenables returned by adapter hooks, but this adapter does -not forward every client return value: only `shadowValidation` and -`staleRecovery` return the counter call's result. A custom DogStatsD client must +not forward every client return value: only `shadowValidation`, +`shadowMismatchLogging`, and `staleRecovery` return the counter call's result. A custom DogStatsD client must handle its own asynchronous delivery failures, including rejected promises. Direct adapter calls do not have DialCache's observer guard. Configure client error handling and shutdown as part of application ownership. @@ -379,6 +382,7 @@ This table uses Prometheus names and types, without the optional prefix. | `dialcache_coalesced_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `scope` | Coalesced requests split by `request_local` or `process` scope | | `dialcache_shadow_validation_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `outcome` | Sampled Redis shadow-job outcomes | | `dialcache_shadow_value_age_histogram` | Histogram | `cache_namespace`, `use_case`, `key_type`, `outcome` | Age in seconds of the validated Redis value at shadow verdict time, recorded for `match` and `mismatch` | +| `dialcache_shadow_mismatch_logging_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `outcome` | Confirmed-mismatch warning attempts: `logged`, `hook_error`, `render_error`, or `logger_error` | | `dialcache_future_timestamp_offset_histogram` | Histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Positive offset in seconds for a valid frame dated after the observing process clock | | `dialcache_stale_recovery_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `outcome` | Classifier-authorized stale-recovery checks: `served`, `miss`, or `deserialization_error` | | `dialcache_stale_recovery_value_age_histogram` | Histogram | `cache_namespace`, `use_case`, `key_type`, `outcome` | Actual return-time age in seconds of a retained value, recorded only for `served` | @@ -542,14 +546,91 @@ Rust's bounded CPU executor), the envelope and mixed-version rollout. ## Confirmed mismatch warnings Shadow metrics remain bounded and contain no cache ids or values. A use case can -separately set `shadow.logMismatches: true` to emit one warning after a terminal -`mismatch` is confirmed. Logging is default-off, does not replace the outcome -metric, and does not activate shadow work without an enabled shadow-outcome hook. +separately opt into one warning after a terminal `mismatch` is confirmed. +Logging is default-off, does not replace the outcome metric, and does not +activate shadow work without an enabled shadow-outcome hook. -The warning contains stable metadata, the logical cache key capped at 2 KiB, -and independently generated native-JSON strings for the cached and source -comparator inputs capped at 8 KiB each. Those fields are value-bearing, and -truncation is not redaction. +The warning contains stable metadata and the logical cache key capped at 2 KiB, +plus what the use case's logging policy selects, such as native-JSON strings for +the cached and source comparator inputs capped at 8 KiB each. Value-bearing +fields are not redacted by truncation. + +The policy can also select the JSON Pointers of the fields that differ, alone or +with the differing values. Each warning attempt, a confirmed mismatch while +logging is on, emits one `shadowMismatchLogging` event with the use-case labels +and an `outcome`: + +| `outcome` | Meaning | +| --- | --- | +| `logged` | Every selected field was built and the logger accepted the warning | +| `hook_error` | The use case's diff hook or the built-in diff failed | +| `render_error` | A value or a hook entry's side had no JSON rendering | +| `logger_error` | The logger failed | + +When several fail, `hook_error` ranks above `render_error`, which ranks above +`logger_error`. When the outcome is not `logged`, the warning carries it as +`loggingOutcome`, except for `logger_error`, which the logger never received. +With logging off nothing is emitted, so the metric never counts mismatches; use +[Shadow outcomes](#shadow-outcomes) for that. The bundled Prometheus and Datadog +adapters export it as `dialcache_shadow_mismatch_logging_counter` and +`dialcache.shadow_mismatch_logging.count`. + + + +The optional `shadowMismatchLogging` adapter hook receives the event. +`hook_error` means `shadowMismatchDiff` threw or returned something other than +an array. `logged` means the logger's `warn` returned, or the promise it +returned resolved; `logger_error` means `warn` threw, or that promise rejected. A +`hook_error` or `render_error` is recorded at once, without waiting for the +logger. Otherwise, for a logger that returns a promise, the event waits for +that promise, and one that never settles emits nothing. + + + + + +Observers and metrics adapters receive an `Event` whose `Kind` is +`shadowMismatchLogging`, with the outcome in `Event.Outcome`; the +`WithShadowOutcomes` hook does not receive it. `hook_error` means +`Operation.MismatchDiff` returned an error or panicked, or the built-in +computation could not parse back a map, slice or `JSONObject` nested more than +10,000 levels deep. A struct whose `MarshalJSON` writes a number outside double +range, or whose JSON nests that deep, has no rendering and reports +`render_error` instead. A Go logger returns nothing, so a `Warn` panic is its +only `logger_error`. + + + + + +Observers receive `Event::ShadowMismatchLogging` with the use-case labels and a +`ShadowMismatchLoggingOutcome`, and `MetricKind::ShadowMismatchLogging` maps it +to the names above. `hook_error` means the `mismatch_diff` hook panicked, or +the built-in computation could not parse a rendering back: one nested 128 or +more levels deep, or, with `serde_json`'s `raw_value` feature on and +`arbitrary_precision` off, a `RawValue` number outside double range. +`render_error` also covers a selected field the operation has no renderer for +and work the bounded CPU executor rejected. A `Logger` panic is the +`logger_error`. + + + + + +Observers receive an event dictionary whose `event` is +`shadowMismatchLogging`, with `cacheNamespace`, `useCase`, `keyType` and +`outcome`. `hook_error` means `shadow_mismatch_diff` raised or returned something +other than a list. `logged` means the logger's `warning` returned, or the +awaitable it returned completed; +`logger_error` means `warning` raised, or that awaitable raised or was +cancelled. DialCache awaits an awaitable the logger returns. A `hook_error` or +`render_error` is recorded at once, without waiting for the logger; otherwise +the event waits for the awaitable, so one that never settles emits nothing. +Python ships no exporter; map the event to +`dialcache_shadow_mismatch_logging_counter` or +`dialcache.shadow_mismatch_logging.count` to match the bundled adapters. + + See [Confirmed mismatch logging](shadow-validation.md#confirmed-mismatch-logging) @@ -601,7 +682,7 @@ raw details remain available through the configured logger; not every metric error or shadow outcome has a matching log entry. The explicitly opted-in confirmed-mismatch warning is a separate value-bearing -log and does not alter the metric schema. +log. Its outcome metric carries labels only. `in_fallback` remains the explicit distinction between cache plumbing and application fallback failures. @@ -623,6 +704,7 @@ Implement `DialCacheMetricsAdapter` and pass it through | `coalesced(labels)` | no | One follower that joined request-local or process-scoped work. | | `shadowValidation(labels)` | no | One terminal sampled-shadow outcome. This hook must be implemented for shadow jobs to execute. | | `observeShadowValueAge(labels, seconds)` | no | Age for shadow match/mismatch verdicts; does not gate admission. | +| `shadowMismatchLogging(labels)` | no | One confirmed-mismatch warning attempt with a bounded `outcome`; emitted only while mismatch logging is on. | | `observeFutureTimestampOffset(labels, seconds)` | no | Positive decoded-frame clock offset; does not change read decisions. | | `staleRecovery(labels)` | no | One authorized recovery outcome; omission does not disable recovery. | | `observeStaleRecoveryValueAge(labels, seconds)` | no | Return-time age for served recovery only. | @@ -638,6 +720,7 @@ Implement `DialCacheMetricsAdapter` and pass it through The root package exports `DialCacheMetricsAdapter` and every associated label, reason, error-kind, layer, scope, and shadow-outcome type, including `ShadowValidationMetricLabels`, `ShadowValidationOutcome`, +`ShadowMismatchLoggingMetricLabels`, `ShadowMismatchLoggingOutcome`, `CompressionMetricLabels`, `CompressionOperationMetricLabels`, and `CompressionOutcome`, `CacheMissReason`, `MissMetricLabels`, `StaleRecoveryMetricLabels`, and `StaleRecoveryOutcome`. @@ -672,6 +755,8 @@ outcomes; `WithRecoveryOutcomes` is an optional recovery observer. Missing the recovery hook does not disable recovery. Avoid forwarding an event both through `WithMetrics` and an outcome hook to the same exporter. +A confirmed-mismatch warning attempt emits a `shadowMismatchLogging` event +whose `Outcome` is `logged`, `hook_error`, `render_error`, or `logger_error`. Use the [generated Go API](api.md) for event and adapter types. Preserve the bounded metric names, labels and units in the catalogue above. @@ -681,6 +766,8 @@ bounded metric names, labels and units in the catalogue above. Implement `Observer` and attach it through the cache builder. It receives typed `Event` values; `MetricKind` maps them to the shared names, labels and units. +Exhaustive matches over `Event` and `MetricKind` need a `ShadowMismatchLogging` +arm. Opt into shadow outcomes through `observes_shadow_outcomes` to admit jobs. The bundled exporters do this; merely logging mismatch warnings does not. @@ -698,7 +785,15 @@ dictionaries use the shared names and camelCase labels, such as `cacheNamespace` an optional `supports("shadowValidation")` method can opt out. The `logger` option accepts a standard Python-compatible warning logger. -See the [generated Python API](api.md) for signatures. +DialCache calls `warning(message, details)`, and when any warning's call returns +an awaitable, DialCache drives it in a task of its own on the event loop. Unless +a `hook_error` or `render_error` was already recorded, a confirmed-mismatch +warning's `shadowMismatchLogging` outcome waits for that awaitable. An awaitable +that never settles keeps its task pending, holding the awaitable and, for a +confirmed mismatch, its namespace, use-case and key-type labels, but neither +compared value. Each such warning retains one task: the cost of counting a +delivery that fails after `warning` returns as `logger_error`. See the +[generated Python API](api.md) for signatures. diff --git a/docs/shadow-validation.md b/docs/shadow-validation.md index 3046053e..7867bf39 100644 --- a/docs/shadow-validation.md +++ b/docs/shadow-validation.md @@ -88,7 +88,7 @@ do not export the same outcome twice. See [observability](observability.md). Use a policy with `.remote_ttl_sec(300).remote_ramp(0.0)` and a -`ShadowPolicy { ramp: Some(5.0), log_mismatches: None }`. Configure +`ShadowPolicy { ramp: Some(5.0), ..ShadowPolicy::default() }`. Configure `shadow_max_in_flight(4)` on the builder and an observer whose `observes_shadow_outcomes` returns true. Bundled exporters opt in. See [observability](observability.md). @@ -121,7 +121,8 @@ A job needs all of the following: - An observer opted into shadow outcomes and available capacity. Missing policy, invalid policy, provider failure, or an omitted outcome hook -cannot start a shadow-only path. `logMismatches` does not enable one either. +cannot start a shadow-only path. Mismatch logging settings do not enable one +either. Serving and shadow cohorts are independent. Equal partial percentages do not select the same keys. A partial shadow ramp therefore does not guarantee every @@ -303,7 +304,8 @@ holding the shadow slot after timeout. Scheduling and shadow deadline timers are unreferenced and do not keep an otherwise idle Node process alive. Detachment uses the event loop, not a worker; -synchronous source, codec, comparator and logger work still consumes it. +synchronous source, codec, comparator and logger work still consumes it, as do +mismatch-warning rendering and `shadowMismatchDiff`. @@ -311,16 +313,23 @@ synchronous source, codec, comparator and logger work still consumes it. Shadow work runs independently of the caller. Native callbacks still consume CPU and can outlive the cache's bounded wait. Give dependency calls finite -context deadlines and bound application concurrency. +context deadlines and bound application concurrency. A job holds its capacity +until its verdict is reported, on the shadow job's goroutine: while the +`WithShadowOutcomes` hook, observers and metrics adapters receive its +`shadowValidation` and `shadowAge` events, and while its mismatch warning is +built and logged, the built-in walk and `Operation.MismatchDiff` included. A +slow callback there therefore shows up as `dropped` jobs beyond capacity. The captured runtime runs detached shadow tasks. Admitted CPU work can retain -capacity even after an async runtime shutdown. Mismatch preview callbacks run -through the bounded CPU executor and can run on worker threads; application -codecs control their own scheduling. +capacity even after an async runtime shutdown. Mismatch-warning content runs +through the bounded CPU executor too: JSON rendering, the built-in walk, +`preview` and the `mismatch_diff` hook can run on worker threads and hold the +job's capacity until they finish. Application codecs control their own +scheduling. @@ -329,6 +338,8 @@ codecs control their own scheduling. The asyncio event loop owns detached shadow work. Keep it alive while work settles. Loaders, codecs and compression execute on that event loop; use native async I/O and application-controlled scheduling for blocking custom work. +Mismatch-warning content, the built-in walk and `shadow_mismatch_diff` +included, also runs there, after the job releases its slot. @@ -371,57 +382,430 @@ mismatch. Logging is default-off and independent of sampling. It can contain cache keys and value previews, so enable it only for data appropriate to your application's logging policy. +Two default-off leaves under `shadow.mismatchLogging` choose what the warning +carries, and the warning is emitted when either is on: + +| Leaf | Values | Adds to the warning | +| --- | --- | --- | +| `diff` | `"off"`, `"paths"`, `"values"` | `"paths"`: where the values differ. `"values"`: also what each side holds there | +| `value` | `true`, `false` | Native JSON of both compared values | + +`diff: "paths"` logs where the values differ but not what they hold, so it is the +level meant to stay on for user data. Member names still appear in the pointers: +an object keyed by identifiers logs those identifiers, exactly as written, +control characters included, so send warnings to a structured or escaping +logger. `cacheKey` still carries the key's arguments as well. This config +excerpt keeps paths-only logging on for a 5% shadow cohort: + -`shadow.logMismatches: true` adds one warning only after confirmed `mismatch`. -It is default-off and independent of sampling. The logger receives the message -`"DialCache shadow validation mismatch"` and an object with `cacheNamespace`, -`useCase`, `keyType`, `outcome: "mismatch"`, and three bounded detail fields: +```ts +shadow: { ramp: 5, mismatchLogging: { diff: "paths" } } +``` + +An invalid leaf in `defaultConfig` throws when `cached()` registers or +`getOrLoad()` runs; see [validation and snapshots](api.md#validation-and-snapshots). -| Field | Content | UTF-8 cap | -| --- | --- | --- | -| `cacheKey` | Logical URN, not the physical Redis key | 2 KiB | -| `cachedValueJson` | Native JSON of the decoded cached snapshot | 8 KiB | -| `sourceValueJson` | Native JSON of the raw source value | 8 KiB | + + + + +```go +Shadow: &dialcache.ShadowPolicy{ + Ramp: dialcache.Ptr(5.0), + MismatchLogging: &dialcache.MismatchLoggingPolicy{Diff: dialcache.Ptr(dialcache.MismatchDiffPaths)}, +}, +``` + +`MismatchLoggingPolicy` holds `Value`, a `*bool`, and `Diff`, a +`*MismatchDiffLevel`: `MismatchDiffOff`, `MismatchDiffPaths` or +`MismatchDiffValues`. `ShadowPolicy.LogMismatches` is the shorthand described +below. Static validation rejects any other `Diff`, and a `JSONPolicy` or +`RawPolicy` overlay uses the JSON names on this page. + + + + + +```rust +ShadowPolicy { + ramp: Some(5.0), + mismatch_logging: Some(MismatchLogging { + diff: Some(MismatchDiff::Paths), + ..MismatchLogging::default() + }), + ..ShadowPolicy::default() +} +``` + +`MismatchLogging` holds `value: Option` and `diff: Option`, +whose variants are `Off`, `Paths` and `Values`. `log_mismatches` is the +shorthand described below, and a runtime overlay uses the JSON names on this +page. + + + + + +```python +shadow={"ramp": 5, "mismatch_logging": {"diff": "paths"}} +``` + +`log_mismatches` is the shorthand described below, and mappings also accept the +camelCase `mismatchLogging` and `logMismatches`. Static defaults reject an +invalid shorthand or leaf with `ConfigError`. + + + +`diff: "values"` and `value: true` log application data and suit an +investigation. + +`shadow.logMismatches` is shorthand for the whole group: `true` means +`{ value: true, diff: "values" }` and `false` means +`{ value: false, diff: "off" }`. A leaf written in the same `shadow` object +overrides what the shorthand expands to. Only then do the leaves merge with the +other configuration layer, runtime over defaults: + +| Written | Effective | +| --- | --- | +| `{ logMismatches: true, mismatchLogging: { value: false } }` | `{ value: false, diff: "values" }` | +| `{ logMismatches: false, mismatchLogging: { diff: "paths" } }` | `{ value: false, diff: "paths" }` | +| Default `mismatchLogging: { diff: "paths" }`, runtime `logMismatches: false` | Both off | +| Default `mismatchLogging: { value: false }`, runtime `logMismatches: true` | `{ value: true, diff: "values" }` | + +To log less than `logMismatches: true`, replace the shorthand with leaves rather +than writing a leaf beside it. A leaf beside `logMismatches: true` leaves the +other expanded leaf on, and DialCache versions before `mismatchLogging` read +only the shorthand, so a service still on such a version keeps logging both +values from the same config. At runtime, an invalid leaf only turns that leaf +off and an invalid shorthand expands to both leaves off; one remote +`config_resolution` covers every invalid input of a call and is recorded only +when a shadow job is admitted. A runtime `mismatchLogging` that is not an +object fails resolution for the whole call, which then runs the loader +uncached. + +The warning carries these fields: + +| Field | Present | Content | UTF-8 cap | +| --- | --- | --- | --- | +| `cacheNamespace`, `useCase`, `keyType`, `outcome: "mismatch"` | Always | Use-case metadata | | +| `cachedValueAgeSeconds` | Always | Age of the cached value at the verdict, as the shadow value-age metric records it; `null` when unavailable | | +| `cacheKey` | Always | Logical URN, not the physical Redis key | 2 KiB | +| `mismatchPaths` | `diff` above `"off"` | RFC 6901 JSON Pointers of the differing fields, at most 64 | 512 bytes each | +| `mismatchPathCount` | `diff` above `"off"` | Number of differing fields before the cut to 64 | | +| `diffJson` | `diff: "values"` | JSON array of `{ path, cached, source }` entries for the same pointers; a side the field lacks is omitted | 8 KiB, 1 KiB per side | +| `cachedValueJson` | `value: true` | Native JSON of the decoded cached snapshot | 8 KiB | +| `sourceValueJson` | `value: true` | Native JSON of the raw source value | 8 KiB | +| `loggingOutcome` | After a failure | `"hook_error"` or `"render_error"`: why fields are `null` | | + + + +The logger's `warn` receives the message +`"DialCache shadow validation mismatch"` and an object with these fields, in +this order. + + + + + +The logger's `Warn` receives the message +`"DialCache shadow validation mismatch"` and a `map[string]any` with these +fields, so they have no order. `cachedValueAgeSeconds` is a `float64`, +`mismatchPaths` a `[]string` and `mismatchPathCount` an `int`; a field that +failed is a nil interface value. Without `WithLogger`, the standard logger +writes each warning on one line through the `log` package, with the fields as +one JSON object, so `mismatchPaths` `[""]` reads differently from `[]` and a +failed field is `null`. In that object, control characters, DEL, U+2028 and +U+2029 are JSON escapes and a byte that is not UTF-8 becomes U+FFFD, so a +member name cannot break the line or send a terminal escape, and the object +still parses. + + + + + +The logger receives `LogEvent::ShadowMismatch` with `ShadowMismatchDetails`: +`cached_value_age_seconds`, `cache_key`, a `diff` group present when `diff` is +above `Off` (`mismatch_paths`, `mismatch_path_count` and, at `Values`, +`diff_json`), a `values` group present when `value` is on +(`cached_value_json`, `source_value_json`), and `logging_outcome`. When the +content work does not complete, as when the CPU executor rejects it, both +groups are `None` whatever the leaves request, and `logging_outcome` is +`RenderError`. +Inside a present group, `None` marks an unavailable field. The default +`LogFacadeLogger` writes the event's `Display`: `name=value` pairs in the order +of the table, without `outcome`, with `namespace`, `cachedValue` and +`sourceValue` in place of `cacheNamespace`, `cachedValueJson` and +`sourceValueJson`, and with unavailable fields left out. + + + + + +The logger's `warning` receives the message +`"DialCache shadow validation mismatch: %s"` and one dict with these fields, in +this order; a field that failed is `None`. + + + +For example, take the `GetUser` use case configured above, with +`mismatchLogging: { diff: "paths" }` added to its shadow policy. It has this +user cached, as decoded from Redis: + +```json +{"id": "u_123", "email": "ana@example.com", "plan": "free", + "roles": ["viewer"], "profile": {"name": "Ana", "locale": "en-US"}} +``` + +The source has changed the user since and now returns: + +```json +{"id": "u_123", "email": "ana@example.com", "plan": "pro", + "roles": ["viewer", "editor"], + "profile": {"name": "Ana", "locale": "en-GB", "timezone": "Europe/London"}} +``` + +After the mismatch is confirmed, the warning carries these fields: + +```jsonc +{ + "cacheNamespace": "users-api", + "useCase": "GetUser", + "keyType": "user_id", + "outcome": "mismatch", + "cachedValueAgeSeconds": 42.5, // varies + "cacheKey": "{users-api:user_id:u_123}#GetUser", + "mismatchPaths": ["/plan", "/profile/locale", "/profile/timezone", "/roles"], + "mismatchPathCount": 4 +} +``` + +`/id`, `/email`, and `/profile/name` match, so they are absent. The paths follow +member names in UTF-16 order, not the order the object was written in, so +`/roles` comes after `/profile`. Only the source has `/profile/timezone`, and +`/roles` changed length, so that array is reported once, whole, rather than +element by element. With `diff: "values"` the same warning also carries +`diffJson`, shown formatted here; the logged string is compact: + +```json +[ + { "path": "/plan", "cached": "free", "source": "pro" }, + { "path": "/profile/locale", "cached": "en-US", "source": "en-GB" }, + { "path": "/profile/timezone", "source": "Europe/London" }, + { "path": "/roles", "cached": ["viewer"], "source": ["viewer", "editor"] } +] +``` + +The `/profile/timezone` entry carries `source` alone because the cached value +has no such member. + +The paths come from the native JSON of both values: each side is rendered once +with the language's native JSON writer, not the Redis serializer, and every +field above derives from that rendering, so a member the rendering leaves out of +`cachedValueJson` stays out of the diff too. The diff is not limited to the +8 KiB that `cachedValueJson` keeps: for a larger value it can show differing +fields from past that point. The two renderings are compared structurally: + +- Object members are compared in UTF-16 code-unit order of their names. A member + present on one side only is reported with that side. +- Arrays of equal length are compared by index. Arrays of different length are + reported once, at the array's own pointer, with both arrays; DialCache does not + align their elements. +- Any other differing pair is reported at its pointer with both sides, and roots + of different kinds are reported at the root pointer `""`. +- Numbers are compared as IEEE doubles, so `1` matches `1.0` and `0` matches + `-0`. `true` and `false` never match a number. +- The comparison descends at most 32 levels: a differing pair of containers 32 + levels down is reported whole at its own pointer. + +An empty `mismatchPaths` means both renderings hold the same JSON, apart from +member order and number precision beyond a double: the comparator saw a +difference that native JSON does not show, such as a native type the rendering +erased, or a custom comparator compared more than the JSON. A side longer than +1 KiB appears in `diffJson` as its clipped JSON text. Clipped fields end with +`...[truncated]` inside their cap. The caps bound what is logged, not the work: +the computation parses and walks both renderings in full, so its CPU time and +memory grow with the rendered size of the values and with the number of members +and elements they hold. + +Each smaller pair below shows one rule: + +| Cached | Source | `mismatchPaths` | Why | +| --- | --- | --- | --- | +| `{"plan":"free"}` | `{"plan":"pro"}` | `["/plan"]` | A changed field | +| `{"tags":["a","b"]}` | `{"tags":["a","c"]}` | `["/tags/1"]` | Same length, so elements are compared by index | +| `{"tags":["a","b"]}` | `{"tags":["a","x","b"]}` | `["/tags"]` | Different length: the whole array, once | +| `{"name":"Ana"}` | `{"name":"Ana","nickname":"A"}` | `["/nickname"]` | Only the source has it, so its `diffJson` entry carries `source` alone | +| `{"name":"Ana","nickname":"A"}` | `{"name":"Ana"}` | `["/nickname"]` | Only the cache has it, as after the source stops returning a field; its `diffJson` entry carries `cached` alone | +| `{"count":1}` | `{"count":"1"}` | `["/count"]` | A number against a string | +| `{"address":{"city":"Paris"}}` | `{"address":null}` | `["/address"]` | An object against `null` is reported whole | +| `{"a/b":1,"m~n":1}` | `{"a/b":2,"m~n":2}` | `["/a~1b","/m~0n"]` | RFC 6901 escapes `/` as `~1` and `~` as `~0` | +| `["a","b"]` | `{"items":["a","b"]}` | `[""]` | Roots of different kinds, at the root pointer | +| Absent | `{"plan":"pro"}` | `[""]` | A cached negative-cache entry is an absent value, not a failure: the source is reported whole at the root, and `cachedValueJson` is `null` | + +A use case can replace the computation with a diff hook, set beside its shadow +comparator. The hook receives the compared values, an absent one included as +the language's own absent marker, and returns the entries, each a `path` with +optional `cached` and `source` sides; a hook that knows only where values +differ returns paths alone. It follows the comparator's rules: synchronous, +deterministic, side-effect-free, non-mutating, and bounded. Its entries are the +use case's own output, so they log whatever it returns. + +Failures log less, never more. A value without a JSON rendering makes its value +field `null`, along with the built-in diff fields. An absent value, the +top-level marker a negative-cache entry stores, is not such a failure: only its +own value field is `null` and the warning carries no `loggingOutcome`. A hook +that fails makes `mismatchPaths`, `mismatchPathCount`, and `diffJson` `null`; +DialCache does not fall back to the built-in computation. A hook value without a +JSON rendering makes `diffJson` alone `null`. Any other failure while building +the fields still logs the warning, with `render_error`: TypeScript writes every +requested field as `null`, and Go, Rust and Python log the metadata alone. +Failures never change the +`shadowValidation` outcome, and the `shadowMismatchLogging` metric reports each +warning attempt; see +[Confirmed mismatch warnings](observability.md#confirmed-mismatch-warnings). + + + +Each side is rendered with `JSON.stringify`, so a member that `toJSON` leaves +out of `cachedValueJson` stays out of the diff too. A cycle, a `bigint`, or a +`toJSON` that throws has no rendering, and a top-level `undefined` is the absent +value. The default comparator tells a string from a `Date`, but both render to +the same JSON: a `Date` that the JSON serializer stored comes back as its ISO +string, so `{"at":"2026-10-24T00:00:00.000Z"}` against +`{ at: new Date("2026-10-24T00:00:00.000Z") }` is a mismatch whose +`mismatchPaths` is `[]`. + +`shadowMismatchDiff` sets the hook; `cached()` and `getOrLoad()` throw a +`TypeError` when it is not a function. A hook that throws, returns a promise or +returns a non-array fails. DialCache logs the first 64 entries as the hook +returns them, so a wrong entry logs a wrong path. This API excerpt assumes a +`fetchUser` source: + +```ts +const getUser = dialcache.cached(fetchUser, { + keyType: "user_id", + useCase: "GetUser", + cacheKey: (userId) => userId, + shadowMismatchDiff: (cached, source) => + cached.email === source.email ? [] : [{ path: "/email" }], +}); +``` -Clipped fields end with `...[truncated]` inside the cap. JSON failure or undefined -output makes that side `null`; the other side is still attempted. Logging does -not compute a diff or reuse the Redis serializer. If detail construction fails, -DialCache still attempts the warning with the four metadata fields; all three detail -fields can be absent. +Rendering, the walk, and the hook run on the event loop after the job releases +its capacity slot. On one Apple-silicon Mac, a 6.3 MB object of 150,000 members +took about 0.12 seconds per warning at `diff: "paths"`, against 0.06 seconds for +the value fields alone. -Set `ShadowPolicy.LogMismatches` to `Ptr(true)`. Confirmed warnings carry bounded -metadata, a logical-key preview (2 KiB), and JSON value previews (8 KiB each). -A failed preview does not prevent the remaining metadata warning. +Each side is rendered with the JSON writer `JSONCodec` uses, not the +operation's `Codec`. It writes maps, slices, `JSONObject` values and scalars by +their kind, without calling their `MarshalJSON`, and renders a struct, with +everything inside it, through `encoding/json`, parsing the output back, so a +struct's `MarshalJSON` runs once. A cycle, a `*big.Int`, a channel, a function, +or a struct whose `MarshalJSON` fails, panics or writes a number outside double +range, or whose JSON nests more than 10,000 levels, has no rendering, and +`Absent` is the absent value. A map, slice or `JSONObject` nested that deep does +render, but the built-in computation cannot parse the rendering back and fails +with `hook_error`. + +`Operation.MismatchDiff`, beside `Comparator`, sets the hook: +`func(cached, source T) ([]MismatchEntry, error)`. A `MismatchEntry` side set +to `Absent` is omitted and nil is JSON `null`, so an entry that only names a +location sets both sides to `Absent`; a nil slice is an empty list. A side +renders like a compared value, except that the side or any value inside it that +implements `json.Marshaler`, such as `json.RawMessage` or a redacting type, +renders through its `MarshalJSON`, whose output is parsed back rather than +copied. A `MarshalJSON` that fails, panics, or writes invalid JSON or a number +outside double range leaves the side with no rendering, and a `*big.Int` side +still has none. The hook fails by returning an error or panicking. + +Rendering, the walk, and the hook run on the shadow job's goroutine after the +verdict and hold the job's shadow capacity until they finish, so while a slow +hook runs, jobs beyond capacity report `dropped`. On one Apple-silicon Mac, a +6.3 MB map of 150,000 members took about 0.6 seconds per warning at +`MismatchDiffPaths`, most of it rendering: the value fields alone took about +0.5 seconds. -Set `ShadowPolicy.log_mismatches` to `Some(true)`. The operation's `preview` -callback renders optional value details. Default JSON previews retain an 8 KiB -prefix while checking serialization; they run through the bounded CPU executor -after confirmation. Admission failure can omit previews while retaining a -metadata warning. The logical-key preview is bounded to 2 KiB. +An operation built with `Operation::new`, or registered with `register`, +renders each side once with `serde_json`, and its default `preview` logs that +JSON. Setting `preview` to `None` logs no values, as before: like an operation +built with `with_codec` or registered with `register_custom`, which has no JSON +rendering, it gets no built-in diff. A custom `preview` replaces the JSON for +the whole warning, so the value fields come from the preview and the diff +fields only from a `mismatch_diff` hook; the built-in walk never reads what a +redacting preview leaves out. A field logged without its renderer, or a +`preview` that returns `None` or panics, is unavailable with `render_error`. +A value whose `Serialize` fails has no rendering, so its value field and the +built-in diff fields are unavailable with `render_error`. The built-in +computation parses each rendering back with `serde_json` and fails with +`hook_error` on one it cannot parse: a rendering nested 128 or more levels +deep, which reaches the parser's recursion limit, or, in a build that enables +`serde_json`'s `raw_value` feature without `arbitrary_precision`, a `RawValue` +holding a number outside double range. Rust's default codec stores no top-level +undefined value, so an absent side does not arise for native values. + +`Operation::mismatch_diff` and `UseCaseBuilder::mismatch_diff` set the hook, +`Fn(&T, &T) -> Vec`, whose entries carry a `path` and optional +`serde_json::Value` sides. A panic is its only failure, and its sides always +render. + +Rendering, the walk, `preview`, and the hook run through the bounded CPU +executor after confirmation and hold the job's shadow capacity until they +finish; they can run on a worker thread. When the executor rejects the work, +or the work panics or is dropped, the warning keeps its metadata, both field +groups are `None`, and it carries `render_error`. On one Apple-silicon Mac, a +6.3 MB map of 150,000 entries took about 0.09 seconds per warning at +`MismatchDiff::Paths` in a release build, against 0.006 seconds for the value +fields alone. + +By default the executor is one process-wide pool of two worker threads and two +queued jobs, which every cache instance also uses for zstd decompression and +for compressing large payloads. While warnings occupy it, a compressed hit +whose decompression it rejects falls through to the source, and a write whose +compression it rejects is skipped. At most `shadow_max_in_flight` warnings per +instance, times the number of instances, contend for it; a custom `Runtime` can +supply its own executor. -Set `shadow={"ramp": 5, "log_mismatches": True}`. Confirmed mismatches emit -bounded native JSON previews (8 KiB) and a logical-key preview (2 KiB). -Serialization failures omit unavailable value previews; warning failures do -not alter cache results. +Each side is rendered once with `json.dumps`, using compact separators, keeping +non-ASCII text, and escaping lone surrogates as `\uXXXX`. Tuples render as +arrays, so an empty `mismatchPaths` can mean a cached list against a source +tuple, which the default comparator tells apart. `NaN`, an infinity, a `set`, +`bytes`, another non-JSON object, a nested `UNDEFINED`, or a cycle has no +rendering, so a `datetime`, `Decimal` or `UUID` in a compared value makes the +diff fields `None` with `render_error` unless `shadow_mismatch_diff` is set. +A top-level `UNDEFINED` is the absent value, and `None` is JSON `null`. + +`shadow_mismatch_diff=`, set beside `shadow_comparator` on `cached()` or +`get_or_load()`, sets the hook. It returns a list of mappings, each with a str +`"path"` and optional `"cached"` and `"source"` sides; a missing key or +`UNDEFINED` omits a side. A hook that raises, returns an awaitable (a coroutine +is closed without running), returns something other than a list, or returns one +of the first 64 entries without a `"path"` key fails. Other entries are logged +as returned, with `str()` of each path. A non-callable value raises +`ConfigError`. + +Rendering, the walk, and the hook run on the event loop after the job releases +its capacity slot. On one Apple-silicon Mac, a 6.3 MB dict of 150,000 members +took about 0.18 seconds per warning at `"paths"`, against 0.04 seconds for the +value fields alone. Truncation is not redaction. Keys and values can contain sensitive application -data. Preview byte limits do not by themselves bound serialization traversal or -CPU cost. Logger failures are isolated from cache behavior. +data. Field byte limits do not bound rendering, parsing, or the walk. Logger +failures are isolated from cache behavior. ## Metrics and shutdown diff --git a/docs/upgrading.md b/docs/upgrading.md index 2ff008f9..e740c418 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -204,6 +204,153 @@ compatible, use a serializer that validates and rejects old shapes, or move to a new identity. Mutually incompatible validating readers can continually replace each other's values while they overlap. +## Shadow mismatch warnings + +`shadow.logMismatches` stays in every port, so no config needs a migration: a +config that sets it to `true` keeps logging every confirmed mismatch. Go and +Rust code that names the changed types needs the edits listed for that language +below. + +`logMismatches` is now shorthand for +`shadow.mismatchLogging: { value: true, diff: "values" }` (`false` for both +leaves off), so its warnings also carry `cachedValueAgeSeconds`, +`mismatchPaths`, `mismatchPathCount`, and `diffJson`. The new fields derive from +the same native-JSON renderings as `cachedValueJson` and `sourceValueJson`, so +a member those renderings omit stays omitted. They are not cut at the 8 KiB +value cap, though: for a larger value, `mismatchPaths` and `diffJson` can log +differing fields past the point where `cachedValueJson` is clipped. Computing +them parses and walks both renderings in full, so each confirmed mismatch costs +more CPU time and memory, growing with the rendered size of the values and the +number of members they hold. A warning can carry up to about 40 KiB more field +bytes than before, counted before any escaping by the logger: 64 pointers of up +to 512 bytes and an 8 KiB `diffJson`. + +To log only the previous fields plus `cachedValueAgeSeconds`, write +`mismatchLogging: { diff: "off" }` beside `logMismatches: true`. To log the +differing fields without values, replace `logMismatches` with +`mismatchLogging: { diff: "paths" }`: a leaf written beside +`logMismatches: true` leaves the other expanded leaf on, and a service still on +an earlier DialCache version reads only `logMismatches` from the same config. +See [Confirmed mismatch logging](shadow-validation.md#confirmed-mismatch-logging) +for how the shorthand and the leaves combine across config layers. + +Each warning attempt also emits a new `shadowMismatchLogging` metric event. The +bundled Datadog adapters' `shadow_mismatch_logging.count` metric is their +longest name, which lowers the longest valid Datadog namespace from 175 to 170 +characters. + + + +`DialCacheKeyConfig.disabled()` now turns logging off with +`shadow.mismatchLogging: { value: false, diff: "off" }` in place of +`shadow.logMismatches: false`. On one Apple-silicon Mac, a 6.3 MB object of +150,000 members took about 0.12 seconds per warning at `diff: "paths"`, against +0.06 seconds for the value fields alone. + + + + + +`ShadowPolicy.LogMismatches` expands to `ShadowPolicy.MismatchLogging` with +`Value` true and `Diff` `MismatchDiffValues`. These changes break code that +names the old shapes: + +- `ResolvedShadow.LogMismatches` is replaced by the effective leaves + `ResolvedShadow.LogValue` and `ResolvedShadow.LogDiff`, and + `LoggingConfigError` now covers every invalid leaf and shorthand. Unkeyed + `ShadowPolicy` and `ResolvedShadow` literals no longer compile. +- `ShadowMismatchLogDetails`, `ShadowMismatchDetails` and + `PreviewShadowLogJSON` are removed, since each value now renders once for + every field; `PreviewShadowLogKey` and the `ShadowLog*` constants remain. +- `FailureIsolatedLogger` returns an already isolated logger unchanged, so a + `Warn` panic stays observable as `logger_error`. +- Observers receive the new `shadowMismatchLogging` event kind, and the + Prometheus adapter registers a twentieth collector, + `dialcache_shadow_mismatch_logging_counter`. + +The standard logger, used without `WithLogger`, now writes the mismatch +warning's details as one JSON object in place of Go's `map[...]` text, so a +pattern that parsed the old lines needs updating. The JSON keeps apart what the +old text ran together: `mismatchPaths` `[""]`, one difference at the root, no +longer reads like `[]`, a pointer that holds a space stays one string, and a +field that failed is `null`. Every line it writes now also escapes control +characters, DEL, U+2028 and U+2029, so an entry that carried a raw newline or +terminal escape stays on one line. Inside the JSON details they are JSON +escapes and a byte that is not UTF-8 becomes U+FFFD, so the details still +parse; in other entries, such as an error's text, they and bytes that are not +UTF-8 are written as Go escape sequences. + +A shadow job now holds its capacity until its verdict is reported: while the +`WithShadowOutcomes` hook, observers and metrics adapters receive its +`shadowValidation` and `shadowAge` events, and while its mismatch warning is +built and logged, the `MismatchDiff` hook and the built-in walk included. +`WithShadowCapacity` therefore bounds that work too, and while a slow hook, +observer or logger runs, jobs beyond capacity report `dropped`. + +The value fields cost what they did before, since each side was already +rendered in full. On one Apple-silicon Mac, a 6.3 MB map of 150,000 members took +about 0.6 seconds per warning at `MismatchDiffPaths`, against about 0.5 seconds +for the value fields alone. + + + + + +`log_mismatches: Some(true)` expands to +`mismatch_logging: Some(MismatchLogging { value: Some(true), diff: Some(MismatchDiff::Values) })`. +These changes break code that names the old shapes: + +- `ShadowPolicy` gained `mismatch_logging`; add `..ShadowPolicy::default()` to + struct literals. `Policy::disabled()` now sets `mismatch_logging` to + `value: Some(false)` and `diff: Some(MismatchDiff::Off)` in place of + `log_mismatches: Some(false)`. +- `ResolvedShadow::log_mismatches` is replaced by `mismatch_value` and + `mismatch_diff`, with `logs_mismatches()`. +- `ShadowMismatchDetails` moved `cached_value_json` and `source_value_json` + into its `values` group, gained `cached_value_age_seconds`, `diff` and + `logging_outcome`, and no longer implements `Eq`. +- `Operation` gained `mismatch_diff` and a private field, so it can no longer + be built with a struct literal; use `Operation::new` or + `Operation::with_codec` and assign fields afterwards. +- `Event::ShadowMismatchLogging` and `MetricKind::ShadowMismatchLogging` are new + variants, so exhaustive matches need an arm. `PrometheusObserver` registers + twenty collectors, adding `dialcache_shadow_mismatch_logging_counter`. + +Only an operation built with `Operation::new` or `register` and left with its +default `preview` feeds the built-in diff. A custom `preview` still replaces the +JSON in the value fields, and a `preview` set to `None` still logs no values, +but neither, nor a `with_codec` or `register_custom` operation, gets difference +fields without a `mismatch_diff` hook. Under `log_mismatches: Some(true)`, such +an operation's warnings carry the fields it cannot build as unavailable, with +`logging_outcome: render_error`, as they do for a `preview` that returns +`None`. Add a hook, or select only leaves the operation can build. + +On one Apple-silicon Mac, a 6.3 MB map of 150,000 entries took about +0.09 seconds per warning at `MismatchDiff::Paths` in a release build, against +0.006 seconds for the value fields alone. That time is spent on the bounded CPU +executor, where previews already ran. By default the executor is one +process-wide pool of two worker threads and two queued jobs, which every cache +instance also uses for zstd decompression and for compressing large payloads. +While warnings occupy it, a compressed hit whose decompression it rejects falls +through to the source, and a write whose compression it rejects is skipped. At +most `shadow_max_in_flight` warnings per instance, times the number of +instances, contend for it; a custom `Runtime` can supply its own executor. + + + + + +`Policy.disabled()` now sets +`shadow={"ramp": 0, "mismatch_logging": {"value": False, "diff": "off"}}` in +place of `"log_mismatches": False`, and a policy merged from runtime overlays +carries the expanded `mismatch_logging` leaves rather than the shorthand. +`cachedValueJson` and `sourceValueJson` now escape lone surrogates as `\uXXXX`, +as `JsonSerializer` does. On one Apple-silicon Mac, a 6.3 MB dict of 150,000 +members took about 0.18 seconds per warning at `"paths"`, on the event loop, +against 0.04 seconds for the value fields alone. + + + ## Metric migrations Miss metrics carry a required bounded `reason`: `value_absent`, `expired`, @@ -238,7 +385,9 @@ Update exhaustive public-union mappings: `tracked_ttl_clamped` is a `ShadowValidationOutcome`. Recovery has `served`, `miss`, and `deserialization_error` outcomes. Custom `miss` handlers now receive `MissMetricLabels`; broader handlers may ignore the additional reason, while -exact label mappings and direct calls need to include it. +exact label mappings and direct calls need to include it. The optional +`shadowMismatchLogging` hook reports a `ShadowMismatchLoggingOutcome`: `logged`, +`hook_error`, `render_error`, or `logger_error`. @@ -247,7 +396,9 @@ exact label mappings and direct calls need to include it. Update exhaustive event/outcome handling when the shared metric catalogue gains a variant. `tracked_ttl_clamped`, shadow `fill_fenced`, and recovery `served`, `miss`, `deserialization_error` must retain their shared meaning and bounded -labels. Native Go adapter contracts are in the [API reference](api.md). +labels. The `shadowMismatchLogging` event kind reports `logged`, `hook_error`, +`render_error`, or `logger_error`. Native Go adapter contracts are in the +[API reference](api.md). @@ -255,15 +406,18 @@ labels. Native Go adapter contracts are in the [API reference](api.md). Update exhaustive matches over public events, metric kinds and outcomes when upgrading. Preserve the shared `tracked_ttl_clamped`, `fill_fenced` and recovery -outcomes, names and units. Native types are in the [API reference](api.md). +outcomes, names and units. The `ShadowMismatchLogging` event reports a +`ShadowMismatchLoggingOutcome`: `logged`, `hook_error`, `render_error`, or +`logger_error`. Native types are in the [API reference](api.md). -Update event dictionary handling for new outcomes and labels. Preserve -`tracked_ttl_clamped`, `fill_fenced` and recovery outcomes with their shared -units. See the [Python API](api.md). +Update event dictionary handling for new outcomes, labels and events, such as +`shadowMismatchLogging` with its `logged`, `hook_error`, `render_error` and +`logger_error` outcomes. Preserve `tracked_ttl_clamped`, `fill_fenced` and +recovery outcomes with their shared units. See the [Python API](api.md). diff --git a/formal/README.md b/formal/README.md index 7befec02..21c0d6cf 100644 --- a/formal/README.md +++ b/formal/README.md @@ -257,7 +257,7 @@ flowchart LR SY["symbolic
make model-check"] EX["exploration
make explore"] GE --> TSP["typescript-parity
make formal-ts"] & GOP["go-parity
make formal-go"] & RSP["rust-parity
make formal-rust"] & PYP["python-parity
make formal-python"] - GE --> TSM["typescript-mutations, 6 shards
make mutations-ts"] & GOM["go-mutations, 10 shards
make mutations-go"] & RSM["rust-mutations, 3 shards
make mutations-rust"] + GE --> TSM["typescript-mutations, 6 shards
make mutations-ts"] & GOM["go-mutations, 15 shards
make mutations-go"] & RSM["rust-mutations, 4 shards
make mutations-rust"] TSM --> TSMM["typescript-mutations-merge
make mutations-merge-ts"] GOM --> GOMM["go-mutations-merge
make mutations-merge-go"] RSM --> RSMM["rust-mutations-merge
make mutations-merge-rust"] @@ -284,7 +284,7 @@ flowchart LR each mode adds to the one before it: `replay` runs `generate`, the four parity lanes and `symbolic`, about 15 minutes; `mutations` adds the three mutation lanes, about 50 minutes; `full`, the default and the weekly mode, - adds `check-models`, about two and a half hours, with its wall clock set by + adds `check-models`, about three hours, with its wall clock set by that job. The parity and mutation lanes wait only for `generate`, each mutation lane merges its shards, and the aggregate requires every selected lane to pass and every other lane to be skipped. It keeps a summary artifact diff --git a/formal/catalogs/behavioral-scenarios.json b/formal/catalogs/behavioral-scenarios.json index 5208b7f7..5c2ac6a7 100644 --- a/formal/catalogs/behavioral-scenarios.json +++ b/formal/catalogs/behavioral-scenarios.json @@ -2631,7 +2631,7 @@ "steps": [ {"input": {"op": "seed","value": 1},"expect": {}}, {"input": {"op": "begin"},"expect": {"calls": [{"status": "value","value": 1}],"reads": 1,"loads": 1,"loaders": 1,"policyCalls": 1}}, - {"input": {"op": "resolve","loader": 0,"value": 2},"expect": {"loads": 2,"shadow": ["mismatch"],"reads": 2,"events": [{"event": "mismatchWarning","cacheNamespace": "urn","useCase": "Behavior","keyType": "id","outcome": "mismatch","cacheKey": "urn:id:1#Behavior","cachedValueJson": "1","sourceValueJson": "2"}]}} + {"input": {"op": "resolve","loader": 0,"value": 2},"expect": {"loads": 2,"shadow": ["mismatch"],"reads": 2,"events": [{"event": "mismatchWarning","cacheNamespace": "urn","useCase": "Behavior","keyType": "id","outcome": "mismatch","cacheKey": "urn:id:1#Behavior","cachedValueAgeSeconds": 0,"mismatchPathsJson": "[\"\"]","mismatchPathCount": 1,"diffJson": "[{\"path\":\"\",\"cached\":1,\"source\":2}]","cachedValueJson": "1","sourceValueJson": "2"}]}} ] }, { diff --git a/formal/catalogs/coverage-witnesses.json b/formal/catalogs/coverage-witnesses.json index 42896bcd..cf5e18b5 100644 --- a/formal/catalogs/coverage-witnesses.json +++ b/formal/catalogs/coverage-witnesses.json @@ -276,7 +276,45 @@ "shadow-confirmation-past-freshness-preserves-payload-and-age", "shadow-confirmation-rollback-keeps-payload-and-reports-offset", "future-dark-c0-reports-offset-and-fills-semantic-miss", - "late-dark-read-reports-offset-without-reviving-job" + "late-dark-read-reports-offset-without-reviving-job", + "fixture:13", + "fixture:14", + "fixture:15", + "warning-fields:value", + "warning-fields:paths", + "warning-fields:values", + "warning-fields:value+paths", + "warning-fields:value+values", + "logging-outcome:logged", + "logging-outcome:hook_error", + "logging-outcome:render_error", + "logging-outcome:logger_error", + "identical-renderings-log-no-paths", + "differing-renderings-log-root-entry", + "unrenderable-source-logs-null", + "hook-error-over-render-error", + "hook-error-over-logger-error", + "render-error-over-logger-error", + "hook-skipped-at-diff-off", + "logger-error-keeps-warning-content", + "shorthand-row:1", + "shorthand-row:2", + "shorthand-row:3", + "shorthand-row:4", + "written-off-leaf:diff", + "written-off-leaf:value", + "invalid-leaf-keeps-valid-leaf", + "invalid-shorthand-keeps-written-leaf", + "shorthand-true-with-value-false-logs-diff-only", + "shorthand-false-with-diff-paths-logs-paths-only", + "runtime-false-shorthand-overrides-default-diff", + "runtime-true-shorthand-overrides-default-value", + "runtime-off-diff-overrides-default-diff", + "runtime-false-value-overrides-default-value", + "hook-error-nulls-diff-fields", + "render-error-nulls-unrenderable-source", + "logger-error-still-delivers-warning", + "hook-error-outranks-both-failures" ], "effects": [ "adapter-frame-last-fresh-millisecond-hits", diff --git a/formal/catalogs/execution.json b/formal/catalogs/execution.json index 35550424..e5773848 100644 --- a/formal/catalogs/execution.json +++ b/formal/catalogs/execution.json @@ -179,7 +179,12 @@ "agesRequireVerdicts", "warningsRequireMismatch", "expiredSourceWorkSkipsRedis", - "invalidLoggingCannotBeCaptured", + "capturedLoggingFollowsLayers", + "noWarningWithLoggingOff", + "oneLoggingOutcomePerWarning", + "warningFieldsMatchCapturedLeaves", + "loggingOutcomePrecedence", + "warningAgeIsVerdictAge", "fillsRequireMissAndAcceptedSource", "c0AcquisitionsRespectFreshness", "budgetsBelongToPendingWork", @@ -1738,6 +1743,300 @@ } } }, + { + "id": "shadow-logging-true-shorthand-expands-off", + "contract": "C60", + "source": "formal/models/kernel/mismatch-logging.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "capturedLoggingFollowsLayers", + "before": "val stands = if (written.shorthand == \"true\") { value: \"true\", diff: DIFF_VALUES }", + "after": "val stands = if (written.shorthand == \"true\") { value: \"false\", diff: DIFF_OFF }", + "reproducer": { + "kind": "exported-regression", + "run": "runtimeTrueShorthandOverridesDefaultValueTest", + "failure": "s.o.shadow == List(\"mismatch\") and s.d.mismatchWarnings == List(LOGGED_EVERYTHING) and s.d.loggingOutcomes == List(\"logged\")", + "family": "mismatch-logging-shorthand-expansion", + "profiles": [ + "shadow" + ], + "exclusions": {} + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M65", + "text": "M65 is the native twin of kernel/mismatch-logging.qnt expandLogging letting a true logMismatches stand for both leaves off: both ports expand a true shorthand to value false and diff off, so a runtime true over a default value false captures nothing and a confirmed mismatch logs no warning." + } + }, + { + "id": "shadow-logging-malformed-shorthand-expands-on", + "contract": "C60", + "source": "formal/models/kernel/mismatch-logging.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "capturedLoggingFollowsLayers", + "before": " else { value: \"false\", diff: DIFF_OFF }\n {", + "after": " else if (written.shorthand == \"false\") { value: \"false\", diff: DIFF_OFF }\n else { value: \"true\", diff: DIFF_VALUES }\n {", + "reproducer": { + "kind": "exported-regression", + "run": "invalidLoggingKeepsComparisonWithoutWarningTest", + "failure": "s.o.shadow == List(\"mismatch\") and s.d.warnings == 0 and s.o.comparisons == 1", + "family": "mismatch-logging-shorthand-expansion", + "profiles": [ + "shadow" + ], + "exclusions": {} + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M08", + "text": "M08 is the native twin of kernel/mismatch-logging.qnt expandLogging letting a malformed logMismatches stand for both leaves on while still marking it invalid: both ports expand it to value true and diff values, so admission records the config_resolution error and the confirmed mismatch warns instead of comparing silently." + } + }, + { + "id": "shadow-logging-written-off-leaf-ignored", + "contract": "C60", + "source": "formal/models/kernel/mismatch-logging.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "capturedLoggingFollowsLayers", + "before": " value: if (written.value != UNWRITTEN) written.value else stands.value,\n diff: if (written.diff != UNWRITTEN) written.diff else stands.diff,", + "after": " value: if (written.value != UNWRITTEN and written.value != \"false\") written.value else stands.value,\n diff: if (written.diff != UNWRITTEN and written.diff != DIFF_OFF) written.diff else stands.diff,", + "reproducer": { + "kind": "exported-regression", + "run": "runtimeOffDiffOverridesDefaultDiffTest", + "failure": "s.o.shadow == List(\"mismatch\") and s.d.configErrors == 0 and s.d.mismatchWarnings == List({ cachedValue: VALUE_ONE, sourceValue: VALUE_TWO, ...WARNING_AT_ZERO }) and s.d.loggingOutcomes == List(\"logged\")", + "family": "mismatch-logging-shorthand-expansion", + "profiles": [ + "shadow" + ], + "exclusions": {} + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M71", + "text": "M71 is the native twin of kernel/mismatch-logging.qnt expandLogging treating a leaf written off as unwritten: both ports drop a written value false or diff off, so a runtime diff off over a default logMismatches true inherits diff values and the warning carries difference fields it should not." + } + }, + { + "id": "shadow-logging-omitted-value-resolves-on", + "contract": "C60", + "source": "formal/models/kernel/mismatch-logging.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "capturedLoggingFollowsLayers", + "before": "logging: { value: merged.value == \"true\",", + "after": "logging: { value: merged.value == \"true\" or merged.value == UNWRITTEN,", + "reproducer": { + "kind": "exported-regression", + "run": "omittedLoggingKeepsMismatchMetricsOnlyTest", + "failure": "s.o.shadow == List(\"mismatch\") and s.d.ages == List(0) and s.d.warnings == 0 and s.d.configErrors == 0 and s.o.comparisons == 1", + "family": "mismatch-logging-leaf-resolution", + "profiles": [ + "shadow" + ], + "exclusions": {} + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M09", + "text": "M09 is the native twin of kernel/mismatch-logging.qnt resolveLogging capturing an unwritten value leaf as on: both ports resolve an omitted value leaf to true, so a shadow policy that writes no logging input warns with both values at a confirmed mismatch." + } + }, + { + "id": "shadow-logging-invalid-diff-leaf-acts-values", + "contract": "C60", + "source": "formal/models/kernel/mismatch-logging.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "capturedLoggingFollowsLayers", + "before": "diff: if (levels.contains(merged.diff)) merged.diff else DIFF_OFF }", + "after": "diff: if (levels.contains(merged.diff)) merged.diff else if (merged.diff == UNWRITTEN) DIFF_OFF else DIFF_VALUES }", + "reproducer": { + "kind": "exported-regression", + "run": "invalidDiffLeafKeepsValueLeafTest", + "failure": "s.d.configErrors == 1 and s.d.mismatchWarnings == List({ cachedValue: VALUE_ONE, sourceValue: VALUE_TWO, ...WARNING_AT_ZERO }) and s.d.loggingOutcomes == List(\"logged\")", + "family": "mismatch-logging-leaf-resolution", + "profiles": [ + "shadow" + ], + "exclusions": {} + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M76", + "text": "M76 is the native twin of kernel/mismatch-logging.qnt resolveLogging capturing an invalid diff leaf as values: both ports resolve a diff leaf of the wrong type to values while recording its config_resolution, so an invalid leaf beside value true adds the entries with their values to the warning." + } + }, + { + "id": "shadow-warning-with-logging-off", + "contract": "C60", + "source": "formal/models/kernel/shadow.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "noWarningWithLoggingOff", + "before": "val warned = if (outcome == \"mismatch\" and logs(job.logging))", + "after": "val warned = if (outcome == \"mismatch\")", + "reproducer": { + "kind": "exported-regression", + "run": "runtimeFalseShorthandOverridesDefaultDiffTest", + "failure": "s.o.shadow == List(\"mismatch\") and s.d.ages == List(0) and s.d.warnings == 0 and s.d.mismatchWarnings == List() and s.d.loggingOutcomes == List() and s.d.configErrors == 0", + "family": "mismatch-warning-opt-in", + "profiles": [ + "shadow", + "dark-layers" + ], + "exclusions": { + "admission": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "shadow-layers": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "shadow-read-deadlines": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy, so the fault adds a warning at each confirmed mismatch, but no run or scheduled invariant of this profile reads the warning count or lists: it claims dark read deadlines and raw ownership, not C60." + } + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M74", + "text": "M74 is the native twin of kernel/shadow.qnt recordVerdict warning at every confirmed mismatch whatever the leaves the job captured: both ports treat both leaves off as opting in, so a runtime false shorthand over a default diff paths still logs a metadata-only warning and a logged outcome." + } + }, + { + "id": "shadow-diff-hook-failure-falls-back-to-walk", + "contract": "C60", + "source": "formal/models/kernel/shadow.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "warningFieldsMatchCapturedLeaves", + "before": "val listed = diffOn and not(hookFails) and not(renderFails)", + "after": "val listed = diffOn and not(renderFails)", + "reproducer": { + "kind": "exported-regression", + "run": "hookErrorNullsDiffFieldsTest", + "failure": "s.o.shadow == List(\"mismatch\") and s.o.calls == List(VALUE_TWO) and s.d.mismatchWarnings == List(failedWith(VALUE_TWO, \"hook_error\")) and s.d.loggingOutcomes == List(\"hook_error\")", + "family": "mismatch-diff-hook-fail-closed", + "profiles": [ + "shadow" + ], + "exclusions": { + "admission": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "dark-layers": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy and no diff hook, so no warning is ever recorded and the mutated warning content or outcome is never kept.", + "shadow-layers": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "shadow-read-deadlines": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy and no diff hook, so no warning is ever recorded and the mutated warning content or outcome is never kept." + } + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M69", + "text": "M69 is the native twin of kernel/shadow.qnt warningOf listing the walk's entries after the diff hook failed: both ports keep hook_error but walk the renderings, so the warning carries the root entry instead of null difference fields." + } + }, + { + "id": "shadow-diff-hook-runs-at-diff-off", + "contract": "C60", + "source": "formal/models/kernel/shadow.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "loggingOutcomePrecedence", + "before": "val hookFails = diffOn and job.diffHook", + "after": "val hookFails = job.diffHook", + "reproducer": { + "kind": "exported-regression", + "run": "hookSkippedWithDiffOffTest", + "failure": "s.d.mismatchWarnings == List({ cachedValue: VALUE_ONE, sourceValue: Shadow::NO_RENDERING, loggingOutcome: \"render_error\", ...WARNING_AT_ZERO }) and s.d.loggingOutcomes == List(\"render_error\")", + "family": "mismatch-diff-hook-fail-closed", + "profiles": [ + "shadow" + ], + "exclusions": { + "admission": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "dark-layers": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy and no diff hook, so no warning is ever recorded and the mutated warning content or outcome is never kept.", + "shadow-layers": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "shadow-read-deadlines": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy and no diff hook, so no warning is ever recorded and the mutated warning content or outcome is never kept." + } + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M70", + "text": "M70 is the native twin of kernel/shadow.qnt warningOf running the throwing diff hook with diff off: both ports call the hook when only value is on and report its failure, so an unrenderable source logs hook_error instead of render_error." + } + }, + { + "id": "shadow-warning-paths-carry-entries", + "contract": "C60", + "source": "formal/models/kernel/shadow.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "warningFieldsMatchCapturedLeaves", + "before": "entries: if (listed and job.logging.diff == DIFF_VALUES) found else List(),", + "after": "entries: if (listed) found else List(),", + "reproducer": { + "kind": "exported-regression", + "run": "valueWithDiffPathsLogsRenderingsAndPathsTest", + "failure": "s.d.mismatchWarnings == List({ cachedValue: VALUE_ONE, sourceValue: VALUE_TWO, diff: \"paths\", count: 1, paths: List(\"\"), ...WARNING_AT_ZERO }) and s.d.loggingOutcomes == List(\"logged\")", + "family": "mismatch-warning-field-groups", + "profiles": [ + "shadow" + ], + "exclusions": { + "admission": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "dark-layers": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy and no diff hook, so no warning is ever recorded and the mutated warning content or outcome is never kept.", + "shadow-layers": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "shadow-read-deadlines": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy and no diff hook, so no warning is ever recorded and the mutated warning content or outcome is never kept." + } + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M75", + "text": "M75 is the native twin of kernel/shadow.qnt warningOf logging the entries at diff paths: both ports treat paths as values, so a warning at diff paths also carries diffJson with each entry's cached and source sides." + } + }, + { + "id": "shadow-logging-render-error-outranks-hook-error", + "contract": "C60", + "source": "formal/models/kernel/shadow.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "loggingOutcomePrecedence", + "before": "loggingOutcome: if (hookFails) \"hook_error\" else if (renderFails) \"render_error\" else \"\"", + "after": "loggingOutcome: if (renderFails) \"render_error\" else if (hookFails) \"hook_error\" else \"\"", + "reproducer": { + "kind": "exported-regression", + "run": "hookErrorOutranksRenderErrorTest", + "failure": "s.d.mismatchWarnings == List(failedWith(Shadow::NO_RENDERING, \"hook_error\")) and s.d.loggingOutcomes == List(\"hook_error\")", + "family": "mismatch-logging-outcome-precedence", + "profiles": [ + "shadow" + ], + "exclusions": { + "admission": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "dark-layers": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy and no diff hook, so no warning is ever recorded and the mutated warning content or outcome is never kept.", + "shadow-layers": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "shadow-read-deadlines": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy and no diff hook, so no warning is ever recorded and the mutated warning content or outcome is never kept." + } + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M72", + "text": "M72 is the native twin of kernel/shadow.qnt warningOf naming render_error before hook_error: both ports check the render failure first, so a throwing diff hook beside an unrenderable source reports render_error on the warning and its outcome." + } + }, + { + "id": "shadow-logging-logger-error-outranks-construction", + "contract": "C60", + "source": "formal/models/kernel/shadow.qnt", + "model": "formal/models/dialcache-shadow-conformance.qnt", + "invariant": "loggingOutcomePrecedence", + "before": "if (warning.loggingOutcome != \"\") warning.loggingOutcome else if (loggerFailed) \"logger_error\" else \"logged\"", + "after": "if (loggerFailed) \"logger_error\" else if (warning.loggingOutcome != \"\") warning.loggingOutcome else \"logged\"", + "reproducer": { + "kind": "exported-regression", + "run": "hookErrorOutranksLoggerErrorTest", + "failure": "s.d.mismatchWarnings == List(failedWith(VALUE_TWO, \"hook_error\")) and s.d.loggingOutcomes == List(\"hook_error\")", + "family": "mismatch-logging-outcome-precedence", + "profiles": [ + "shadow" + ], + "exclusions": { + "admission": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "dark-layers": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy and no diff hook, so no warning is ever recorded and the mutated warning content or outcome is never kept.", + "shadow-layers": "Settles its jobs through the undiagnosed lifecycle (Shadow::settleRead, settleLoad or settle), never through recordVerdict, so no warning or logging outcome is ever built.", + "shadow-read-deadlines": "Records verdicts through recordVerdict under the constant Shadow::LOGGING_OFF policy and no diff hook, so no warning is ever recorded and the mutated warning content or outcome is never kept." + } + }, + "nativeMutants": { + "kind": "mapped", + "mutant": "M73", + "text": "M73 is the native twin of kernel/shadow.qnt loggingOutcomeOf letting a failed logger decide before the construction failure the warning names: both ports record logger_error when the logger also fails, while the warning keeps hook_error." + } + }, { "id": "redis-protocol-inclusive-fence", "contract": "C33", diff --git a/formal/catalogs/feature-coverage.json b/formal/catalogs/feature-coverage.json index 5b9a971f..f1f96edc 100644 --- a/formal/catalogs/feature-coverage.json +++ b/formal/catalogs/feature-coverage.json @@ -526,6 +526,7 @@ "C58", "C59", "C60", + "W10", "X01" ], "cases": [ @@ -546,7 +547,13 @@ "C58.request-follower-trail", "C58.late-rejection-not-recounted", "C59.compression-read-telemetry", - "C60.logging-explicit-off" + "C60.logging-explicit-off", + "C60.warning-field-groups", + "C60.shorthand-table", + "C60.invalid-input-isolation", + "C60.logging-outcomes", + "C60.outcome-precedence", + "W10.mismatch-paths" ], "nativeCases": [ "X01.prometheus-schema", @@ -584,7 +591,7 @@ "test": "captures an immutable default policy snapshot when the use case is registered", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Asserts that the TypeScript binding captures an immutable default policy snapshot when the use case is registered.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" } ], "go": [ @@ -723,25 +730,25 @@ "test": "accepts disabled and exact-maximum static stale-on-error policy", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Asserts that the TypeScript binding accepts disabled and exact-maximum static stale-on-error policy.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" }, { "test": "rejects unsupported runtime TTLs before cache or serialization work", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Asserts that the TypeScript binding rejects unsupported runtime TTLs before cache or serialization work.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" }, { "test": "rejects $0 in the public config constructor", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Malformed policy objects, layer maps, boolean leaves, and shadow shapes reject at public configuration construction.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" }, { "test": "rejects a malformed static defaultConfig with $0 at registration", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Invalid static TTLs, ramps, recovery limits, and policy shapes reject before reserving the use case; a corrected policy may reuse the same name.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" }, { "test": "rejects invalid fallbackTimeoutMs value %s without reserving the use case", @@ -773,13 +780,13 @@ "test": "TestPolicyStaticValidation", "path": "go/internal/testsuite/policy_test.go", "scope": "Malformed option types and combinations reject; the maximum supported remote TTL/read budget is accepted and fractional-second static TTL is rejected.", - "sha256": "91439ed1fd62de50007188a14751d45a2c5424be384d6ac393a8226644793c57" + "sha256": "2550c855d7a0c923f691704e317bce25d3f08b590199654ffc8c451736cffd55" }, { "test": "TestRuntimePolicyFailureScopes", "path": "go/internal/testsuite/policy_test.go", "scope": "Malformed invocation-wide leaves fail resolution; invalid layer/recovery leaves retain the other active layers and their failure categories.", - "sha256": "91439ed1fd62de50007188a14751d45a2c5424be384d6ac393a8226644793c57" + "sha256": "2550c855d7a0c923f691704e317bce25d3f08b590199654ffc8c451736cffd55" } ], "adaptation": "Native number and optional-value encodings differ; each binding must distinguish absent, explicitly disabled, malformed, and boundary values.", @@ -801,31 +808,31 @@ "test": "preserves requestLocal omission until baseline and runtime config are merged", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Asserts that the TypeScript binding preserves requestLocal omission until baseline and runtime config are merged.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" }, { "test": "preserves coalesce omission until baseline and runtime config are merged", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Asserts that the TypeScript binding preserves coalesce omission until baseline and runtime config are merged.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" }, { "test": "clones the supplied shadow policy", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Asserts that the TypeScript binding clones the supplied shadow policy.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" }, { "test": "preserves stale-on-error omission and explicit disable values", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Absent stale recovery remains absent while explicit zero and positive maximum age retain their distinct construction values.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" }, { "test": "preserves shadow omission and explicit kill-switch values", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Absent shadow remains absent; an empty shadow overlay and explicit zero ramp/false logging remain distinct sparse construction values.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" } ], "go": [ @@ -833,7 +840,7 @@ "test": "TestPolicySparseResolutionAndSnapshots", "path": "go/internal/testsuite/policy_test.go", "scope": "Nested runtime leaves merge independently, omitted leaves inherit, and captured nested pointer-valued defaults survive caller mutation.", - "sha256": "91439ed1fd62de50007188a14751d45a2c5424be384d6ac393a8226644793c57" + "sha256": "2550c855d7a0c923f691704e317bce25d3f08b590199654ffc8c451736cffd55" } ], "adaptation": "Eager defaulting can erase inherited leaves even when ordinary effective-policy examples pass.", @@ -861,7 +868,7 @@ "test": "accepts a positive shadowMaxInFlight", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Asserts that the TypeScript binding accepts a positive shadowMaxInFlight.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" }, { "test": "allows localMaxSize zero to disable local storage", @@ -879,7 +886,7 @@ "test": "rejects invalid shadowMaxInFlight value %s", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Null, zero, negative, fractional, non-finite, and unsafe shadow capacities reject under the positive safe-integer constraint.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" } ], "go": [ @@ -1026,7 +1033,7 @@ "test": "TestObserverIsolationAndBoundedNativeJSONLogs", "path": "go/metrics_test.go", "scope": "Native observer panic/error and logger panics are isolated; JSON/key previews obey UTF-8 byte caps and unsupported/cyclic values have null previews.", - "sha256": "48f62015286274c2c3a0d778907a3ba6ced73c9bcd9e19b438cf5e372d1a41f3" + "sha256": "55657dc4165d0c5fd40b796a904472d6b65402b2b0704f593cf4c618d3ba3639" } ], "adaptation": "Observability must remain diagnostic even when both the primary operation and logger fail.", @@ -1331,21 +1338,21 @@ "test": "preserves the public collector schema", "path": "typescript/test/prometheus.test.ts", "scope": "Asserts that the TypeScript binding preserves the public collector schema.", - "sha256": "1c081c00ca8642c8ed5e7a3fd5a5faa2d5d06d333f99c2fb0832ead0057c1602" + "sha256": "b459867a7646b3096985a9e42318af7eaf1d4a6a84c64b6e7145837558100296" }, { "test": "exports every bounded error category without rewriting labels", "path": "typescript/test/prometheus.test.ts", "scope": "Asserts that the TypeScript binding exports every bounded error category without rewriting labels.", - "sha256": "1c081c00ca8642c8ed5e7a3fd5a5faa2d5d06d333f99c2fb0832ead0057c1602" + "sha256": "b459867a7646b3096985a9e42318af7eaf1d4a6a84c64b6e7145837558100296" } ], "go": [ { "test": "TestPrometheusWireSchemaMatchesTypeScriptBinding", "path": "go/metrics_test.go", - "scope": "All nineteen Go collector names, help text, label order, kinds and bucket boundaries are checked against the TS binding schema.", - "sha256": "48f62015286274c2c3a0d778907a3ba6ced73c9bcd9e19b438cf5e372d1a41f3" + "scope": "All twenty Go collector names, help text, label order, kinds and bucket boundaries are checked against the TS binding schema.", + "sha256": "55657dc4165d0c5fd40b796a904472d6b65402b2b0704f593cf4c618d3ba3639" } ], "adaptation": "Exporter schema changes break dashboards even when portable event semantics still pass; current Go schema check derives its expectation from TS source, not Quint.", @@ -1365,19 +1372,19 @@ "test": "keeps default DialCache construction detached from Prometheus", "path": "typescript/test/prometheus.test.ts", "scope": "Asserts that the TypeScript binding keeps default DialCache construction detached from Prometheus.", - "sha256": "1c081c00ca8642c8ed5e7a3fd5a5faa2d5d06d333f99c2fb0832ead0057c1602" + "sha256": "b459867a7646b3096985a9e42318af7eaf1d4a6a84c64b6e7145837558100296" }, { "test": "registers collectors only in the caller-owned registry", "path": "typescript/test/prometheus.test.ts", "scope": "Asserts that the TypeScript binding registers collectors only in the caller-owned registry.", - "sha256": "1c081c00ca8642c8ed5e7a3fd5a5faa2d5d06d333f99c2fb0832ead0057c1602" + "sha256": "b459867a7646b3096985a9e42318af7eaf1d4a6a84c64b6e7145837558100296" }, { "test": "reuses existing collectors when multiple adapters share a registry", "path": "typescript/test/prometheus.test.ts", "scope": "Asserts that the TypeScript binding reuses existing collectors when multiple adapters share a registry.", - "sha256": "1c081c00ca8642c8ed5e7a3fd5a5faa2d5d06d333f99c2fb0832ead0057c1602" + "sha256": "b459867a7646b3096985a9e42318af7eaf1d4a6a84c64b6e7145837558100296" } ], "go": [ @@ -1385,7 +1392,7 @@ "test": "TestPrometheusReuseAndConflictIsolation", "path": "go/metrics_test.go", "scope": "Compatible adapters reuse counters and histograms in one caller registry; incompatible pre-existing metrics cause no partial collector registration.", - "sha256": "48f62015286274c2c3a0d778907a3ba6ced73c9bcd9e19b438cf5e372d1a41f3" + "sha256": "55657dc4165d0c5fd40b796a904472d6b65402b2b0704f593cf4c618d3ba3639" } ], "adaptation": "Registry ownership and collector reuse avoid global pollution and duplicate collectors in multi-instance deployments.", @@ -1405,19 +1412,19 @@ "test": "rejects a pre-registered legacy miss collector before registering anything", "path": "typescript/test/prometheus.test.ts", "scope": "Asserts that the TypeScript binding rejects a pre-registered legacy miss collector before registering anything.", - "sha256": "1c081c00ca8642c8ed5e7a3fd5a5faa2d5d06d333f99c2fb0832ead0057c1602" + "sha256": "b459867a7646b3096985a9e42318af7eaf1d4a6a84c64b6e7145837558100296" }, { "test": "rejects exemplar-enabled collectors before registering anything", "path": "typescript/test/prometheus.test.ts", "scope": "Asserts that the TypeScript binding rejects exemplar-enabled collectors before registering anything.", - "sha256": "1c081c00ca8642c8ed5e7a3fd5a5faa2d5d06d333f99c2fb0832ead0057c1602" + "sha256": "b459867a7646b3096985a9e42318af7eaf1d4a6a84c64b6e7145837558100296" }, { "test": "reuses a pre-registered current-schema miss collector", "path": "typescript/test/prometheus.test.ts", "scope": "Asserts that the TypeScript binding reuses a pre-registered current-schema miss collector.", - "sha256": "1c081c00ca8642c8ed5e7a3fd5a5faa2d5d06d333f99c2fb0832ead0057c1602" + "sha256": "b459867a7646b3096985a9e42318af7eaf1d4a6a84c64b6e7145837558100296" } ], "go": [ @@ -1425,19 +1432,19 @@ "test": "TestPrometheusReuseAndConflictIsolation", "path": "go/metrics_test.go", "scope": "Compatible adapters reuse counters and histograms in one caller registry; incompatible pre-existing metrics cause no partial collector registration.", - "sha256": "48f62015286274c2c3a0d778907a3ba6ced73c9bcd9e19b438cf5e372d1a41f3" + "sha256": "55657dc4165d0c5fd40b796a904472d6b65402b2b0704f593cf4c618d3ba3639" }, { "test": "TestPrometheusExplicitBindingsPreserveExistingObservations", "path": "go/metrics_test.go", "scope": "Explicit compatible externally created collectors preserve pre-existing counter and histogram observations when bound by an adapter.", - "sha256": "48f62015286274c2c3a0d778907a3ba6ced73c9bcd9e19b438cf5e372d1a41f3" + "sha256": "55657dc4165d0c5fd40b796a904472d6b65402b2b0704f593cf4c618d3ba3639" }, { "test": "TestPrometheusBindingsRejectIncompatibleOrUnregisteredCollectors", "path": "go/metrics_test.go", "scope": "Explicit collector bindings reject malformed schema, conflicting registry state or unregistered collector identities.", - "sha256": "48f62015286274c2c3a0d778907a3ba6ced73c9bcd9e19b438cf5e372d1a41f3" + "sha256": "55657dc4165d0c5fd40b796a904472d6b65402b2b0704f593cf4c618d3ba3639" } ], "adaptation": "Go requires explicit collector bindings/schema because client_golang cannot expose empty histogram bucket schema; no destructive compatibility probing is permitted.", @@ -1459,19 +1466,19 @@ "test": "maps every DialCache metric to the default metric namespace with exact names, values, and tags", "path": "typescript/test/datadog.test.ts", "scope": "Asserts that the TypeScript binding maps every DialCache metric to the default metric namespace with exact names, values, and tags.", - "sha256": "8b50a81e6baa8d3d1eb4854cb9d84d83cc48493bca4efc0b656e0f0714f66d65" + "sha256": "2c953d1fa3f861dd2d036105856a4d8bfed72887b8fc77f0c7354ababff7a747" }, { "test": "emits hot-shots histogram datagrams when explicitly selected", "path": "typescript/test/datadog.test.ts", "scope": "Asserts that the TypeScript binding emits hot-shots histogram datagrams when explicitly selected.", - "sha256": "8b50a81e6baa8d3d1eb4854cb9d84d83cc48493bca4efc0b656e0f0714f66d65" + "sha256": "2c953d1fa3f861dd2d036105856a4d8bfed72887b8fc77f0c7354ababff7a747" }, { "test": "requires an explicit observation metric type", "path": "typescript/test/datadog.test.ts", "scope": "Asserts that the TypeScript binding requires an explicit observation metric type.", - "sha256": "8b50a81e6baa8d3d1eb4854cb9d84d83cc48493bca4efc0b656e0f0714f66d65" + "sha256": "2c953d1fa3f861dd2d036105856a4d8bfed72887b8fc77f0c7354ababff7a747" } ], "go": [ @@ -1479,7 +1486,7 @@ "test": "TestDatadogMetricNamesUnitsAndLabels", "path": "go/metrics_test.go", "scope": "Native DogStatsD method/name/unit tables, representative bounded labels, histogram mode, namespace restrictions and nil-client validation are checked.", - "sha256": "48f62015286274c2c3a0d778907a3ba6ced73c9bcd9e19b438cf5e372d1a41f3" + "sha256": "55657dc4165d0c5fd40b796a904472d6b65402b2b0704f593cf4c618d3ba3639" } ], "adaptation": "Different observation modes have different backend semantics and require a namespace migration; current schema source remains native integration code.", @@ -1500,19 +1507,19 @@ "test": "never emits cache ids, arguments, Redis keys, or raw errors as Datadog tags", "path": "typescript/test/datadog.test.ts", "scope": "Asserts that the TypeScript binding never emits cache ids, arguments, Redis keys, or raw errors as Datadog tags.", - "sha256": "8b50a81e6baa8d3d1eb4854cb9d84d83cc48493bca4efc0b656e0f0714f66d65" + "sha256": "2c953d1fa3f861dd2d036105856a4d8bfed72887b8fc77f0c7354ababff7a747" }, { "test": "enforces Datadog's 200-character final metric-name limit", "path": "typescript/test/datadog.test.ts", "scope": "Asserts that the TypeScript binding enforces Datadog's 200-character final metric-name limit.", - "sha256": "8b50a81e6baa8d3d1eb4854cb9d84d83cc48493bca4efc0b656e0f0714f66d65" + "sha256": "2c953d1fa3f861dd2d036105856a4d8bfed72887b8fc77f0c7354ababff7a747" }, { "test": "validates every required DogStatsD method before emitting metrics", "path": "typescript/test/datadog.test.ts", "scope": "Asserts that the TypeScript binding validates every required DogStatsD method before emitting metrics.", - "sha256": "8b50a81e6baa8d3d1eb4854cb9d84d83cc48493bca4efc0b656e0f0714f66d65" + "sha256": "2c953d1fa3f861dd2d036105856a4d8bfed72887b8fc77f0c7354ababff7a747" } ], "go": [ @@ -1520,7 +1527,7 @@ "test": "TestDatadogMetricNamesUnitsAndLabels", "path": "go/metrics_test.go", "scope": "Native DogStatsD method/name/unit tables, representative bounded labels, histogram mode, namespace restrictions and nil-client validation are checked.", - "sha256": "48f62015286274c2c3a0d778907a3ba6ced73c9bcd9e19b438cf5e372d1a41f3" + "sha256": "55657dc4165d0c5fd40b796a904472d6b65402b2b0704f593cf4c618d3ba3639" } ], "adaptation": "Typed Go methods replace TS duck-typed method validation, but namespace and cardinality constraints remain observable integration contracts.", @@ -1540,19 +1547,19 @@ "test": "byte-clamps keys and JSON without splitting UTF-8 sequences", "path": "typescript/test/shadow-log-json.test.ts", "scope": "Asserts that the TypeScript binding byte-clamps keys and JSON without splitting UTF-8 sequences.", - "sha256": "97d2a80d23bbe76574f97b0f569307a203279de4fa770a253603648325801467" + "sha256": "2dab2b5558563125f731410b170a41c8c216ff28d54a012b3744ce387c99213f" }, { "test": "catches user JSON hook failures", "path": "typescript/test/shadow-log-json.test.ts", "scope": "Asserts that the TypeScript binding catches user JSON hook failures.", - "sha256": "97d2a80d23bbe76574f97b0f569307a203279de4fa770a253603648325801467" + "sha256": "2dab2b5558563125f731410b170a41c8c216ff28d54a012b3744ce387c99213f" }, { "test": "returns null independently for values that native JSON cannot serialize", "path": "typescript/test/shadow-log-json.test.ts", "scope": "Asserts that the TypeScript binding returns null independently for values that native JSON cannot serialize.", - "sha256": "97d2a80d23bbe76574f97b0f569307a203279de4fa770a253603648325801467" + "sha256": "2dab2b5558563125f731410b170a41c8c216ff28d54a012b3744ce387c99213f" } ], "go": [ @@ -1560,7 +1567,7 @@ "test": "TestObserverIsolationAndBoundedNativeJSONLogs", "path": "go/metrics_test.go", "scope": "Native observer panic/error and logger panics are isolated; JSON/key previews obey UTF-8 byte caps and unsupported/cyclic values have null previews.", - "sha256": "48f62015286274c2c3a0d778907a3ba6ced73c9bcd9e19b438cf5e372d1a41f3" + "sha256": "55657dc4165d0c5fd40b796a904472d6b65402b2b0704f593cf4c618d3ba3639" } ], "adaptation": "C60 covers whether a warning happens; bounded formatting and native JSON failures are separate observability behavior.", @@ -2046,7 +2053,7 @@ "test": "rejects the removed public ramp sampler override", "path": "typescript/test/dialcache-config-ramp.test.ts", "scope": "Asserts that the TypeScript binding rejects the removed public ramp sampler override.", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa" + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed" }, { "test": "rejects the renamed urnPrefix option for untyped callers", diff --git a/formal/catalogs/fixture-recipes.json b/formal/catalogs/fixture-recipes.json index 63781820..6f301a44 100644 --- a/formal/catalogs/fixture-recipes.json +++ b/formal/catalogs/fixture-recipes.json @@ -131,7 +131,9 @@ "configErrors": true, "fallbackErrors": true, "warnings": true, - "futureOffsets": true + "futureOffsets": true, + "mismatchWarnings": true, + "loggingOutcomes": true } }, "dark-layers": { @@ -2109,6 +2111,91 @@ "id": "futureLateDarkReadKeepsTimeoutAndReportsOffsetTest", "regression": "futureLateDarkReadKeepsTimeoutAndReportsOffsetTest", "projection": "shadow" + }, + { + "id": "trueShorthandWithValueFalseLogsDiffOnlyTest", + "regression": "trueShorthandWithValueFalseLogsDiffOnlyTest", + "projection": "shadow" + }, + { + "id": "falseShorthandWithDiffPathsLogsPathsOnlyTest", + "regression": "falseShorthandWithDiffPathsLogsPathsOnlyTest", + "projection": "shadow" + }, + { + "id": "runtimeFalseShorthandOverridesDefaultDiffTest", + "regression": "runtimeFalseShorthandOverridesDefaultDiffTest", + "projection": "shadow" + }, + { + "id": "runtimeTrueShorthandOverridesDefaultValueTest", + "regression": "runtimeTrueShorthandOverridesDefaultValueTest", + "projection": "shadow" + }, + { + "id": "runtimeOffDiffOverridesDefaultDiffTest", + "regression": "runtimeOffDiffOverridesDefaultDiffTest", + "projection": "shadow" + }, + { + "id": "runtimeFalseValueOverridesDefaultValueTest", + "regression": "runtimeFalseValueOverridesDefaultValueTest", + "projection": "shadow" + }, + { + "id": "hookErrorNullsDiffFieldsTest", + "regression": "hookErrorNullsDiffFieldsTest", + "projection": "shadow" + }, + { + "id": "renderErrorNullsUnrenderableSourceTest", + "regression": "renderErrorNullsUnrenderableSourceTest", + "projection": "shadow" + }, + { + "id": "loggerErrorStillDeliversWarningTest", + "regression": "loggerErrorStillDeliversWarningTest", + "projection": "shadow" + }, + { + "id": "hookErrorOutranksBothFailuresTest", + "regression": "hookErrorOutranksBothFailuresTest", + "projection": "shadow" + }, + { + "id": "hookErrorOutranksRenderErrorTest", + "regression": "hookErrorOutranksRenderErrorTest", + "projection": "shadow" + }, + { + "id": "renderErrorOutranksLoggerErrorTest", + "regression": "renderErrorOutranksLoggerErrorTest", + "projection": "shadow" + }, + { + "id": "hookSkippedWithDiffOffTest", + "regression": "hookSkippedWithDiffOffTest", + "projection": "shadow" + }, + { + "id": "identicalRenderingsLogNoPathsTest", + "regression": "identicalRenderingsLogNoPathsTest", + "projection": "shadow" + }, + { + "id": "invalidDiffLeafKeepsValueLeafTest", + "regression": "invalidDiffLeafKeepsValueLeafTest", + "projection": "shadow" + }, + { + "id": "invalidValueLeafKeepsDiffLeafTest", + "regression": "invalidValueLeafKeepsDiffLeafTest", + "projection": "shadow" + }, + { + "id": "invalidShorthandKeepsDiffLeafBesideItTest", + "regression": "invalidShorthandKeepsDiffLeafBesideItTest", + "projection": "shadow" } ] }, diff --git a/formal/catalogs/go-parity.json b/formal/catalogs/go-parity.json index b19a004e..245317b4 100644 --- a/formal/catalogs/go-parity.json +++ b/formal/catalogs/go-parity.json @@ -8,32 +8,32 @@ "counts": "Inventory counts establish accounting completeness only; they are not semantic coverage percentages or proof of parity." }, "inputs": { - "semanticCasesSha256": "f5f7ee357201b343eb00f8f9c783c92af0d722709137edc0f55e0a52b3cdec0a", - "executionSha256": "29a412f87329347840f000ea44a6e95332464c0b01b92b4e41c615af3c9157cd", - "sourceAuditDispositionsSha256": "e05fcb0da204544920a87eeb98611cc697c7f2652fead527e409595010d09377", - "featureCoverageSha256": "2d4e47ec18688c911ce4d6bda780edc8ee654069aa5704e15a39cbae35655ba7", - "coverageWitnessesSha256": "2b2d5d57daacba3ecd3637f5a917d58b0b63fcc88893e651cd602cdba1ef4244", - "profilesSha256": "9df8285820e4fff65f6a7753885bbb192c09f3f476eac8f9c98d130454ae1a28" + "semanticCasesSha256": "380273598e1d956a3f9fce270cc84424b342e456139b279bd56576bd3b837f65", + "executionSha256": "eaf117c3e8630013cc8df0524f84af440c470f42e8711ab4dd07b931161c784f", + "sourceAuditDispositionsSha256": "c95ce03c30df574246e5943e2d69e084ed809b5d3e2a3d0f0abf5a2123bc8aaa", + "featureCoverageSha256": "3f33092274a0046c2d793790b1b4999021021e018f47ae65c15d3a0313776fd2", + "coverageWitnessesSha256": "745cebececddbfa97912621ada94072d648e8e60e8b3eb013b91eefd16f352a1", + "profilesSha256": "888a522f8ad97862ebcb0b5fe363ec7bec60367fa42c60e1623141814b1991b3" }, "inventory": { - "behavioralCases": 244, - "withRequiredGeneratedWitnesses": 244, + "behavioralCases": 249, + "withRequiredGeneratedWitnesses": 249, "withoutRequiredGeneratedWitnesses": [], - "wireCases": 22, - "sourceFiles": 27, - "sourceDeclarations": 772, - "casesWithScheduledChecks": 266, - "requiredWitnesses": 497, - "casesWithQuintRegressionReplay": 192, + "wireCases": 23, + "sourceFiles": 28, + "sourceDeclarations": 878, + "casesWithScheduledChecks": 272, + "requiredWitnesses": 535, + "casesWithQuintRegressionReplay": 198, "casesWithQuintGeneratedVectors": 22, "withoutModelDrivenReplay": [], "quintModels": 32, - "scheduledInvariants": 253, - "scheduledRegressions": 468, + "scheduledInvariants": 258, + "scheduledRegressions": 489, "quintGeneratedProtocolVectors": 1343, "quintGeneratedInvalidationVectors": 288, - "caseReferencedWitnesses": 343, - "exportedRegressionHistories": 358 + "caseReferencedWitnesses": 378, + "exportedRegressionHistories": 379 }, "cases": [ { @@ -2958,6 +2958,78 @@ "kind": "portable-behavior", "gaps": [] }, + { + "id": "C60.warning-field-groups", + "kind": "portable-behavior", + "group": "shadow", + "gaps": [], + "evidence": { + "typescript": ".formal-traces/ts-replay.json", + "go": ".formal-traces/go-replay.jsonl", + "nativeGo": [], + "scope": "Only the linked finite scenarios, witnesses, scoped checks and explicit native boundaries; not universal case proof." + } + }, + { + "id": "C60.shorthand-table", + "kind": "portable-behavior", + "group": "shadow", + "gaps": [], + "evidence": { + "typescript": ".formal-traces/ts-replay.json", + "go": ".formal-traces/go-replay.jsonl", + "nativeGo": [], + "scope": "Only the linked finite scenarios, witnesses, scoped checks and explicit native boundaries; not universal case proof." + } + }, + { + "id": "C60.invalid-input-isolation", + "kind": "portable-behavior", + "group": "shadow", + "gaps": [], + "evidence": { + "typescript": ".formal-traces/ts-replay.json", + "go": ".formal-traces/go-replay.jsonl", + "nativeGo": [], + "scope": "Only the linked finite scenarios, witnesses, scoped checks and explicit native boundaries; not universal case proof." + } + }, + { + "id": "C60.logging-outcomes", + "kind": "portable-behavior", + "group": "shadow", + "gaps": [], + "evidence": { + "typescript": ".formal-traces/ts-replay.json", + "go": ".formal-traces/go-replay.jsonl", + "nativeGo": [], + "scope": "Only the linked finite scenarios, witnesses, scoped checks and explicit native boundaries; not universal case proof." + } + }, + { + "id": "C60.outcome-precedence", + "kind": "portable-behavior", + "group": "shadow", + "gaps": [], + "evidence": { + "typescript": ".formal-traces/ts-replay.json", + "go": ".formal-traces/go-replay.jsonl", + "nativeGo": [], + "scope": "Only the linked finite scenarios, witnesses, scoped checks and explicit native boundaries; not universal case proof." + } + }, + { + "id": "W10.mismatch-paths", + "kind": "wire-protocol", + "group": "wire-protocol", + "gaps": [], + "evidence": { + "typescript": ".formal-traces/ts-replay.json", + "go": ".formal-traces/go-replay.jsonl", + "nativeGo": [], + "scope": "Only the linked finite scenarios, witnesses, scoped checks and explicit native boundaries; not universal case proof." + } + }, { "id": "C40.fenced-candidate", "group": "recovery", @@ -3317,14 +3389,14 @@ "sourceInventory": [ { "path": "typescript/src/config.ts", - "sha256": "3ece031f00acee5067f18db6a8eab35a725fd87e1525c60dfd9c578741953cbf", + "sha256": "e8fafc6f9452a40c212150e492ccc8223bb1e13efe94cb5708be9b0fca6cc392", "candidateGoFiles": [ "go/policy.go", "go/api.go" ], "mappingReview": { "status": "reviewed-mapping-not-execution", - "rationale": "Static configuration and callback contracts map to typed time.Duration Policy, functional Option constructors and Operation[T]. ParsePolicy exposes the seconds-based JSON input; pointer leaves and functional options preserve omission versus explicit false/zero. ResolvePolicy applies the sparse overlay and validated feature defaults.", + "rationale": "Static configuration and callback contracts map to typed time.Duration Policy, functional Option constructors and Operation[T]. ParsePolicy exposes the seconds-based JSON input; pointer leaves and functional options preserve omission versus explicit false/zero. ResolvePolicy applies the sparse overlay and validated feature defaults. The shadow.mismatchLogging leaves map to ShadowPolicy.MismatchLogging with a typed MismatchDiffLevel, and the logMismatches shorthand stays a *bool that expands to them.", "nativeAdaptations": [ "B01", "B02" @@ -3338,9 +3410,11 @@ "ParsePolicy", "ValidatePolicy", "SnapshotPolicy", - "ResolvePolicy" + "ResolvePolicy", + "MismatchLoggingPolicy", + "MismatchDiffLevel" ], - "sha256": "bdedffc9a21ec47ca8660e1e116469ff7b14d96d56646d6f181bd59d11281e93" + "sha256": "436567152ae6c4c3d8fe9169bfbf27ac30c80b8cedcf43dcd5bd81daf88085dd" }, { "path": "go/api.go", @@ -3348,9 +3422,10 @@ "Option", "Operation", "Logger", - "RecoveryPredicate" + "RecoveryPredicate", + "MismatchEntry" ], - "sha256": "103a9b9fd955dcb3644295cd2e8cb41769f7b2ea0dce5cd61f97e452a2498c1f" + "sha256": "45ae467fbcef94cb47ebb9e9912f06c81ee03529ad63a108e91db017cd3a1f7d" } ] } @@ -3387,14 +3462,14 @@ }, { "path": "typescript/src/datadog.ts", - "sha256": "078ee508e948dcf997b556e6ab34d6c5e779c057dd588f2b7b16dee40fcf2c41", + "sha256": "6582c119ce95f473ceede6c5807575e82b01df9a9e6255698379273878c26ea0", "candidateGoFiles": [ "go/metrics_datadog.go", "go/metrics.go" ], "mappingReview": { "status": "reviewed-mapping-not-execution", - "rationale": "DogStatsD integration maps event kinds, labels, units and suffixes through an application-owned DogStatsDClient. Go uses an interface returning errors and an explicit constructor; FailureIsolatedObserver prevents exporter failures from changing cache outcomes.", + "rationale": "DogStatsD integration maps event kinds, labels, units and suffixes through an application-owned DogStatsDClient. Go uses an interface returning errors and an explicit constructor; FailureIsolatedObserver prevents exporter failures from changing cache outcomes. shadowMismatchLogging maps to shadow_mismatch_logging.count, which lowers the longest namespace to 170 characters.", "nativeAdaptations": [ "B01", "X01" @@ -3408,7 +3483,7 @@ "NewDatadogMetrics", "ObserveEvent" ], - "sha256": "84f56821e1234a7ae77569c1429d2385a13bedb1ce7d1a5186a1d808ebe6809b" + "sha256": "d398ab0e2690ae6d6614d4f50d8de7d324ce85717bd2fba1158afa30aac440bb" }, { "path": "go/metrics.go", @@ -3417,14 +3492,14 @@ "metricLabels", "metricValue" ], - "sha256": "fd97d9f8bcf1c9c15c616c974bd3d658239813c33874a78ca8fbb516caf9e6cf" + "sha256": "9066cceb94a24bdc514eca2a9a7090a7d437cc2b18bc389bd0d83e97ff7b5448" } ] } }, { "path": "typescript/src/dialcache.ts", - "sha256": "90e83928537806591b4de1553c81dc542bb59615150b90d5897d929532e15702", + "sha256": "88e1e9bdf70bba085f4c8f2861cfac89ec021a5af318e21dc94e9f54c75b583c", "candidateGoFiles": [ "go/cache.go", "go/bindings.go", @@ -3436,7 +3511,7 @@ ], "mappingReview": { "status": "reviewed-mapping-not-execution", - "rationale": "The public orchestration is divided into Cache lifecycle, Cached registration, GetOrLoad execution, source/read deadlines and detached shadow work. GetOrLoad snapshots inputs before awaiting policy; singleFlight owns leaders/followers. Go contexts, generics, returned errors and recovered callback panics replace TypeScript overloads, promises and exceptions; default diagnostic logging lives in an internal package, while public logger callbacks and inspection remain root-package bindings.", + "rationale": "The public orchestration is divided into Cache lifecycle, Cached registration, GetOrLoad execution, source/read deadlines and detached shadow work. GetOrLoad snapshots inputs before awaiting policy; singleFlight owns leaders/followers. Go contexts, generics, returned errors and recovered callback panics replace TypeScript overloads, promises and exceptions; default diagnostic logging lives in an internal package, while public logger callbacks and inspection remain root-package bindings. The warning's leaves come from the invocation's ResolvedShadow, Operation.MismatchDiff replaces the built-in walk, and warnShadowMismatch emits one shadowMismatchLogging event; a Warn panic is the Go logger_error.", "nativeAdaptations": [ "B01", "B02", @@ -3491,9 +3566,10 @@ "symbols": [ "darkSource", "scheduleShadow", - "runShadow" + "runShadow", + "warnShadowMismatch" ], - "sha256": "3224f4c9cab7dd5a7038bd8b0ccd03158209a486a814bcd0250041526760d81d" + "sha256": "8d29f1a6fdd546e31e08a67056696a4c9c6689479b93a2269f1c74b7cba8050a" }, { "path": "go/api.go", @@ -3502,14 +3578,14 @@ "Operation", "Event" ], - "sha256": "103a9b9fd955dcb3644295cd2e8cb41769f7b2ea0dce5cd61f97e452a2498c1f" + "sha256": "45ae467fbcef94cb47ebb9e9912f06c81ee03529ad63a108e91db017cd3a1f7d" }, { "path": "go/internal/logging/logging.go", "symbols": [ "Logger" ], - "sha256": "c26312555cbc1980a8af2a714ae58c0dfc8182624c44635f905d4147f5db7624" + "sha256": "70900d17293a0931fbcd11811bc389563d6dad9788b69fa01fb539e9f44a2428" } ] } @@ -3537,7 +3613,7 @@ "CallbackPanicError", "ErrNoRemote" ], - "sha256": "103a9b9fd955dcb3644295cd2e8cb41769f7b2ea0dce5cd61f97e452a2498c1f" + "sha256": "45ae467fbcef94cb47ebb9e9912f06c81ee03529ad63a108e91db017cd3a1f7d" }, { "path": "go/redis_adapter.go", @@ -3552,7 +3628,7 @@ }, { "path": "typescript/src/index.ts", - "sha256": "bbf222cec750670240b0475f1e4ab8a071d86fee2a10c3e1d30b8d314d38be5d", + "sha256": "4f3f9fbd2d017e343ecd59bec34f8e5f55212af598ff3fee3877adb8a8ef8a12", "candidateGoFiles": [ "go/api.go", "go/bindings.go", @@ -3563,7 +3639,7 @@ ], "mappingReview": { "status": "reviewed-mapping-not-execution", - "rationale": "The TypeScript root barrel becomes exported declarations in one Go module/package. Cache construction, typed wrapper registration, policy, key and codec helpers are available directly; JavaScript import aliases and package subpath export validation do not have separate Go implementations.", + "rationale": "The TypeScript root barrel becomes exported declarations in one Go module/package. Cache construction, typed wrapper registration, policy, key and codec helpers are available directly; JavaScript import aliases and package subpath export validation do not have separate Go implementations. The mismatch-logging config, hook entry and level types are root declarations (MismatchLoggingPolicy, MismatchDiffLevel, MismatchEntry); metric labels are Event data.", "nativeAdaptations": [ "B01" ], @@ -3576,7 +3652,7 @@ "Codec", "Remote" ], - "sha256": "103a9b9fd955dcb3644295cd2e8cb41769f7b2ea0dce5cd61f97e452a2498c1f" + "sha256": "45ae467fbcef94cb47ebb9e9912f06c81ee03529ad63a108e91db017cd3a1f7d" }, { "path": "go/bindings.go", @@ -3597,7 +3673,7 @@ "symbols": [ "Policy" ], - "sha256": "bdedffc9a21ec47ca8660e1e116469ff7b14d96d56646d6f181bd59d11281e93" + "sha256": "436567152ae6c4c3d8fe9169bfbf27ac30c80b8cedcf43dcd5bd81daf88085dd" }, { "path": "go/protocol.go", @@ -3611,7 +3687,7 @@ "symbols": [ "JSONCodec" ], - "sha256": "387879a07365e10bddbabd21e61cab72102cd5e81f2afb980c33e3d4dbeb6268" + "sha256": "e4f36c61f72610eb57ad838c2359c28d664bd4c5c0f37e10b8b6789072b567f3" } ] } @@ -3675,7 +3751,7 @@ "EscapeRawPayload", "firstZstdFrame" ], - "sha256": "387879a07365e10bddbabd21e61cab72102cd5e81f2afb980c33e3d4dbeb6268" + "sha256": "e4f36c61f72610eb57ad838c2359c28d664bd4c5c0f37e10b8b6789072b567f3" } ] } @@ -3714,7 +3790,7 @@ "TimerClock", "DeferredExecutor" ], - "sha256": "103a9b9fd955dcb3644295cd2e8cb41769f7b2ea0dce5cd61f97e452a2498c1f" + "sha256": "45ae467fbcef94cb47ebb9e9912f06c81ee03529ad63a108e91db017cd3a1f7d" } ] } @@ -3751,7 +3827,7 @@ "policyTTLMS", "ValidatePolicy" ], - "sha256": "bdedffc9a21ec47ca8660e1e116469ff7b14d96d56646d6f181bd59d11281e93" + "sha256": "436567152ae6c4c3d8fe9169bfbf27ac30c80b8cedcf43dcd5bd81daf88085dd" } ] } @@ -3802,7 +3878,7 @@ }, { "path": "go/api.go", - "sha256": "103a9b9fd955dcb3644295cd2e8cb41769f7b2ea0dce5cd61f97e452a2498c1f", + "sha256": "45ae467fbcef94cb47ebb9e9912f06c81ee03529ad63a108e91db017cd3a1f7d", "symbols": [ "systemClock", "newSystemClock" @@ -3835,7 +3911,7 @@ "symbols": [ "ResolvePolicy" ], - "sha256": "bdedffc9a21ec47ca8660e1e116469ff7b14d96d56646d6f181bd59d11281e93" + "sha256": "436567152ae6c4c3d8fe9169bfbf27ac30c80b8cedcf43dcd5bd81daf88085dd" } ] } @@ -3875,7 +3951,7 @@ "symbols": [ "runShadow" ], - "sha256": "3224f4c9cab7dd5a7038bd8b0ccd03158209a486a814bcd0250041526760d81d" + "sha256": "8d29f1a6fdd546e31e08a67056696a4c9c6689479b93a2269f1c74b7cba8050a" }, { "path": "go/codec.go", @@ -3883,7 +3959,7 @@ "CompressPayload", "DecompressPayload" ], - "sha256": "387879a07365e10bddbabd21e61cab72102cd5e81f2afb980c33e3d4dbeb6268" + "sha256": "e4f36c61f72610eb57ad838c2359c28d664bd4c5c0f37e10b8b6789072b567f3" }, { "path": "go/redis_adapter.go", @@ -4010,13 +4086,13 @@ }, { "path": "typescript/src/internal/runtime-config.ts", - "sha256": "7fbf42ce9d346f30d65b431b546649d9592405073dd5c900559f1e4e3c6fb703", + "sha256": "ff7542dcc67816453eeb1c85e54f00aa7d32aad61278891ee431ff9ae1ebb37f", "candidateGoFiles": [ "go/policy.go" ], "mappingReview": { "status": "reviewed-mapping-not-execution", - "rationale": "Runtime policy parsing, leaf-wise inheritance, invocation snapshots and feature-local invalidation map to ParsePolicy, SnapshotPolicy and ResolvePolicy. Nil/Absent whole-provider results inherit while explicit null leaves remain malformed; constructor and wrapper validation occur through Go APIs.", + "rationale": "Runtime policy parsing, leaf-wise inheritance, invocation snapshots and feature-local invalidation map to ParsePolicy, SnapshotPolicy and ResolvePolicy. Nil/Absent whole-provider results inherit while explicit null leaves remain malformed; constructor and wrapper validation occur through Go APIs. Each layer's logMismatches shorthand expands before the leaf-wise merge in ResolvePolicy, and each mismatch-logging leaf validates independently into one LoggingConfigError.", "nativeAdaptations": [ "B01", "B02" @@ -4028,23 +4104,27 @@ "ParsePolicy", "SnapshotPolicy", "ResolvePolicy", - "ValidatePolicy" + "ValidatePolicy", + "expandMismatchLogging", + "resolveValueLeaf", + "resolveDiffLeaf" ], - "sha256": "bdedffc9a21ec47ca8660e1e116469ff7b14d96d56646d6f181bd59d11281e93" + "sha256": "436567152ae6c4c3d8fe9169bfbf27ac30c80b8cedcf43dcd5bd81daf88085dd" } ] } }, { "path": "typescript/src/internal/shadow-log-json.ts", - "sha256": "6fae44c08e79ea2fe63bed8de91c621c1ec59672a415608ade3acf63b18ac91a", + "sha256": "ddab3366b00b83f19c796f3e457b9a12dc422e61f3f2b1b1d1c41731785687d5", "candidateGoFiles": [ "go/metrics.go", - "go/codec.go" + "go/codec.go", + "go/shadow_log.go" ], "mappingReview": { "status": "reviewed-mapping-not-execution", - "rationale": "Mismatch previews use the Go JSON compatibility writer, byte-bounded UTF-8 output and the same truncation marker. Native type/prototype representation is a codec binding; failures return no preview and cannot suppress the shadow verdict or caller result.", + "rationale": "Warning content renders each compared value once with the Go JSON compatibility writer; the built-in walk runs over the parsed renderings with the same member order, array rule, root cases, depth cap (deep equality past it) and caps, materializing only the kept pointers up to their clamp, and the MismatchDiff hook's entries take the same clamps. Byte-bounded UTF-8 output uses the same truncation marker; a failure nulls only the fields it feeds and cannot suppress the verdict or caller result. Native type representation is a codec binding (B03): Go's decoder fails a rendering nested past 10,000 levels as hook_error. Hook entry sides honor json.Marshaler at every depth, as TypeScript sides honor toJSON, while the codec stored encoding reaches MarshalJSON only through a struct.", "nativeAdaptations": [ "B03", "X01", @@ -4054,18 +4134,56 @@ { "path": "go/metrics.go", "symbols": [ - "PreviewShadowLogKey", - "PreviewShadowLogJSON", - "ShadowMismatchLogDetails" + "PreviewShadowLogKey" ], - "sha256": "fd97d9f8bcf1c9c15c616c974bd3d658239813c33874a78ca8fbb516caf9e6cf" + "sha256": "9066cceb94a24bdc514eca2a9a7090a7d437cc2b18bc389bd0d83e97ff7b5448" }, { "path": "go/codec.go", "symbols": [ "appendJSON" ], - "sha256": "387879a07365e10bddbabd21e61cab72102cd5e81f2afb980c33e3d4dbeb6268" + "sha256": "e4f36c61f72610eb57ad838c2359c28d664bd4c5c0f37e10b8b6789072b567f3" + }, + { + "path": "go/shadow_log.go", + "symbols": [ + "renderShadowMismatchLog", + "walkJSON", + "runMismatchHook", + "appendMismatchEntries" + ], + "sha256": "f7ddb1edd7e9b3b7319818f0adf1ecaf42a023d18aa8427a0602709f65921ff1" + } + ] + } + }, + { + "path": "typescript/src/internal/thenable.ts", + "sha256": "359a72f7b3b3c05979dca303bc7450f49282505c0ff7dfd987519e7550767919", + "candidateGoFiles": [ + "go/deadline.go", + "go/metrics.go" + ], + "mappingReview": { + "status": "reviewed-mapping-not-execution", + "rationale": "TypeScript application callbacks under a synchronous contract (comparators, the stale recovery predicate, the mismatch hook) and observers (loggers, metrics adapters) can still return a promise-like; isPromiseLike detects one and fulfills awaits it so a rejection is consumed and observer outcomes can wait for settlement. Go callbacks and observers are synchronous calls with no settlement to observe: callSafely turns a callback panic into CallbackPanicError, and isolatedLogger and FailureIsolatedObserver recover a logger or observer panic, which are the Go counterparts of a rejected promise.", + "goBindings": [ + { + "path": "go/deadline.go", + "symbols": [ + "callSafely" + ], + "sha256": "4c96942152e994adc4171e4d30de2de8be83ab6b3a33ed62793cc86819ce2b97" + }, + { + "path": "go/metrics.go", + "symbols": [ + "isolatedLogger", + "FailureIsolatedLogger", + "FailureIsolatedObserver" + ], + "sha256": "9066cceb94a24bdc514eca2a9a7090a7d437cc2b18bc389bd0d83e97ff7b5448" } ] } @@ -4100,7 +4218,7 @@ }, { "path": "typescript/src/metrics.ts", - "sha256": "6c2607ec512c10362f1906ff082799d8b770f34cb3d61d5523dba908349d90d8", + "sha256": "92a149a709de87362ca58059df966d812d23a5915615a57f012ec7ecbc574f09", "candidateGoFiles": [ "go/api.go", "go/metrics.go", @@ -4109,7 +4227,7 @@ ], "mappingReview": { "status": "reviewed-mapping-not-execution", - "rationale": "The backend-neutral Event stream expresses portable diagnostic outcomes, attribution and durations. Optional shadow/recovery hooks preserve admission semantics; FailureIsolatedObserver and FailureIsolatedLogger contain native exporter/logging failures. Exporter schemas are separately reviewed bindings.", + "rationale": "The backend-neutral Event stream expresses portable diagnostic outcomes, attribution and durations. Optional shadow/recovery hooks preserve admission semantics; FailureIsolatedObserver and FailureIsolatedLogger contain native exporter/logging failures. Exporter schemas are separately reviewed bindings. shadowMismatchLogging is an Event kind with the outcome in Event.Outcome, and warnDelivered reports a Warn panic as logger_error.", "nativeAdaptations": [ "B01", "B02", @@ -4123,16 +4241,17 @@ "Option", "Logger" ], - "sha256": "103a9b9fd955dcb3644295cd2e8cb41769f7b2ea0dce5cd61f97e452a2498c1f" + "sha256": "45ae467fbcef94cb47ebb9e9912f06c81ee03529ad63a108e91db017cd3a1f7d" }, { "path": "go/metrics.go", "symbols": [ "MetricsAdapter", "FailureIsolatedObserver", - "FailureIsolatedLogger" + "FailureIsolatedLogger", + "warnDelivered" ], - "sha256": "fd97d9f8bcf1c9c15c616c974bd3d658239813c33874a78ca8fbb516caf9e6cf" + "sha256": "9066cceb94a24bdc514eca2a9a7090a7d437cc2b18bc389bd0d83e97ff7b5448" }, { "path": "go/engine.go", @@ -4148,7 +4267,7 @@ "symbols": [ "shadowEvent" ], - "sha256": "3224f4c9cab7dd5a7038bd8b0ccd03158209a486a814bcd0250041526760d81d" + "sha256": "8d29f1a6fdd546e31e08a67056696a4c9c6689479b93a2269f1c74b7cba8050a" } ] } @@ -4184,13 +4303,13 @@ }, { "path": "typescript/src/prometheus.ts", - "sha256": "56c11cd6fba362c4b0ab16b222cd5785a37bdc81fdde67ae920557ba05ed5ec7", + "sha256": "b0f0d19fd359a45debf11ecd02af016cae13476c5e5b32bf5c63414870247c63", "candidateGoFiles": [ "go/metrics_prometheus.go" ], "mappingReview": { "status": "reviewed-mapping-not-execution", - "rationale": "PrometheusMetrics preserves metric names, labels, histogram units and buckets using client_golang. Existing individually registered collectors require explicit schema-bearing bindings plus descriptor and registry-identity validation; this replaces prom-client registry introspection and makes histogram-schema knowledge an explicit caller precondition.", + "rationale": "PrometheusMetrics preserves metric names, labels, histogram units and buckets using client_golang. Existing individually registered collectors require explicit schema-bearing bindings plus descriptor and registry-identity validation; this replaces prom-client registry introspection and makes histogram-schema knowledge an explicit caller precondition. dialcache_shadow_mismatch_logging_counter is a twentieth collector schema.", "nativeAdaptations": [ "B01", "X01" @@ -4206,7 +4325,7 @@ "NewPrometheusMetricsWithBindings", "ObserveEvent" ], - "sha256": "c29b357dff3917acb3dbfe3912a74f6c3b7a6312b9d110577bd30e960bc5d87e" + "sha256": "1027779b913bb890cd06ffe314d80840ce67c522bc12dad9b6e12b4224a9d340" } ] } @@ -4233,7 +4352,7 @@ "Remote", "ReadBudget" ], - "sha256": "103a9b9fd955dcb3644295cd2e8cb41769f7b2ea0dce5cd61f97e452a2498c1f" + "sha256": "45ae467fbcef94cb47ebb9e9912f06c81ee03529ad63a108e91db017cd3a1f7d" }, { "path": "go/protocol.go", @@ -4309,7 +4428,7 @@ "Codec", "ContextCodec" ], - "sha256": "103a9b9fd955dcb3644295cd2e8cb41769f7b2ea0dce5cd61f97e452a2498c1f" + "sha256": "45ae467fbcef94cb47ebb9e9912f06c81ee03529ad63a108e91db017cd3a1f7d" }, { "path": "go/codec.go", @@ -4321,7 +4440,7 @@ "SemanticEqual", "validateJSONScalarStrings" ], - "sha256": "387879a07365e10bddbabd21e61cab72102cd5e81f2afb980c33e3d4dbeb6268" + "sha256": "e4f36c61f72610eb57ad838c2359c28d664bd4c5c0f37e10b8b6789072b567f3" } ] } @@ -4968,7 +5087,8 @@ "go/metrics.go", "go/feature_replay_test.go", "go/behavior_replay_test.go", - "go/internal/testsuite/codec_test.go" + "go/internal/testsuite/codec_test.go", + "go/shadow_warning_test.go" ], "nativeAdaptations": [ "B02", @@ -5118,7 +5238,9 @@ "go/metrics_prometheus.go", "go/metrics_datadog.go", "go/metrics_test.go", - "go/api_boundary_test.go" + "go/api_boundary_test.go", + "go/shadow_log.go", + "go/shadow_log_test.go" ], "nativeAdaptations": [ "B02", diff --git a/formal/catalogs/mutations.json b/formal/catalogs/mutations.json index 94de1070..c09130dc 100644 --- a/formal/catalogs/mutations.json +++ b/formal/catalogs/mutations.json @@ -236,13 +236,13 @@ "id": "M08", "case": "C60.invalid-logging-policy", "description": "Enable mismatch logging after a malformed runtime logging flag", - "rationale": "TypeScript typescript/src/dialcache.ts DialCache.resolveShadowLogging returns true after recording the config_resolution error for a malformed runtime logging flag, and Go go/policy.go ResolvePolicy turns LogMismatches on when the flag is invalid. A malformed logMismatches value therefore enables mismatch logging instead of leaving it off.", + "rationale": "TypeScript typescript/src/internal/runtime-config.ts expandMismatchLogging and Go go/policy.go expandMismatchLogging expand a malformed logMismatches shorthand to both mismatch-logging leaves on while still marking it invalid, so admission records the config_resolution error. A malformed logMismatches value therefore enables mismatch logging instead of leaving it off.", "typescript": { "edits": [ { - "path": "typescript/src/dialcache.ts", - "before": "this.recordError(key, CacheLayer.REMOTE, \"config_resolution\");\n return false;\n }\n return configuredLogMismatches;", - "after": "this.recordError(key, CacheLayer.REMOTE, \"config_resolution\");\n return true;\n }\n return configuredLogMismatches;" + "path": "typescript/src/internal/runtime-config.ts", + "before": ": { value: false, diff: \"off\", invalid: true };", + "after": ": { value: true, diff: \"values\", invalid: true };" } ], "requiredDetections": [ @@ -255,8 +255,8 @@ "edits": [ { "path": "go/policy.go", - "before": "resolved.Shadow.LogMismatches = flag\n\t\t\tresolved.Shadow.LoggingConfigError = !valid", - "after": "resolved.Shadow.LogMismatches = flag || !valid\n\t\t\tresolved.Shadow.LoggingConfigError = !valid" + "before": "leaves.value, leaves.diff, leaves.invalid = false, MismatchDiffOff, true", + "after": "leaves.value, leaves.diff, leaves.invalid = true, MismatchDiffValues, true" } ], "requiredDetections": [ @@ -269,13 +269,13 @@ "id": "M09", "case": "C60.logging-default-off", "description": "Enable mismatch logging when its flag is omitted", - "rationale": "TypeScript typescript/src/dialcache.ts DialCache.resolveShadowLogging returns true when the runtime logging flag is omitted, and Go go/policy.go ResolvePolicy starts the resolved policy with LogMismatches on. Mismatch logging is therefore enabled by default instead of off.", + "rationale": "TypeScript typescript/src/internal/runtime-config.ts resolveValueLeaf and Go go/policy.go resolveValueLeaf resolve an omitted value leaf to true, so a shadow group that writes neither logMismatches nor mismatchLogging logs whole values. Mismatch logging is therefore enabled by default instead of off.", "typescript": { "edits": [ { - "path": "typescript/src/dialcache.ts", - "before": "if (configuredLogMismatches === undefined) {\n return false;", - "after": "if (configuredLogMismatches === undefined) {\n return true;" + "path": "typescript/src/internal/runtime-config.ts", + "before": "function resolveValueLeaf(leaf: unknown): boolean | null {\n if (leaf === undefined) {\n return false;", + "after": "function resolveValueLeaf(leaf: unknown): boolean | null {\n if (leaf === undefined) {\n return true;" } ], "requiredDetections": [ @@ -288,8 +288,8 @@ "edits": [ { "path": "go/policy.go", - "before": "resolved := ResolvedPolicy{Coalesce: true, RemoteReadTimeout: defaults.RemoteReadTimeout}", - "after": "resolved := ResolvedPolicy{Coalesce: true, RemoteReadTimeout: defaults.RemoteReadTimeout, Shadow: ResolvedShadow{LogMismatches: true}}" + "before": "func resolveValueLeaf(leaf any) (bool, bool) {\n\tif IsAbsent(leaf) {\n\t\treturn false, true", + "after": "func resolveValueLeaf(leaf any) (bool, bool) {\n\tif IsAbsent(leaf) {\n\t\treturn true, true" } ], "requiredDetections": [ @@ -1845,8 +1845,8 @@ "edits": [ { "path": "go/shadow.go", - "before": "if err != nil {\n\t\tverdict = shadowVerdict{outcome: \"timeout\"}\n\t}", - "after": "if err != nil {\n\t\tverdict = shadowVerdict{outcome: \"timeout\"}\n\t\tx.cache.mu.Lock()\n\t\tif x.cache.shadows[x.key] == f {\n\t\t\tdelete(x.cache.shadows, x.key)\n\t\t}\n\t\tx.cache.mu.Unlock()\n\t}" + "before": "if err != nil {\n\t\tverdict = shadowVerdict[T]{outcome: \"timeout\"}\n\t}", + "after": "if err != nil {\n\t\tverdict = shadowVerdict[T]{outcome: \"timeout\"}\n\t\tx.cache.mu.Lock()\n\t\tif x.cache.shadows[x.key] == f {\n\t\t\tdelete(x.cache.shadows, x.key)\n\t\t}\n\t\tx.cache.mu.Unlock()\n\t}" } ], "requiredDetections": [ @@ -2078,6 +2078,399 @@ "portable" ] } + }, + { + "id": "M65", + "case": "C60.logging-enabled", + "description": "Leave mismatch logging off when its flag is true", + "rationale": "TypeScript typescript/src/internal/runtime-config.ts expandMismatchLogging and Go go/policy.go expandMismatchLogging expand a true logMismatches shorthand to both mismatch-logging leaves off. Enabling mismatch logging with the flag therefore emits no warning for a confirmed mismatch.", + "typescript": { + "edits": [ + { + "path": "typescript/src/internal/runtime-config.ts", + "before": "? { value: true, diff: \"values\" }", + "after": "? { value: false, diff: \"off\" }" + } + ], + "requiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/policy.go", + "before": "leaves.value, leaves.diff = true, MismatchDiffValues\n", + "after": "leaves.value, leaves.diff = false, MismatchDiffOff\n" + } + ], + "requiredDetections": [ + "generated", + "portable" + ] + } + }, + { + "id": "M66", + "case": "W10.mismatch-paths", + "description": "Walk arrays of different length over their common prefix instead of reporting them once at the array pointer", + "rationale": "TypeScript typescript/src/internal/shadow-log-json.ts walkJson, Go go/shadow_log.go walkJSON and Rust rust/src/protocol/mismatch.rs Walk::walk drop the array rule's length check and recurse by index over the shorter array. Arrays of different length are therefore reported at the differing indices of their common prefix, and an inserted or appended element is not reported at all, instead of one entry at the array's own pointer carrying both arrays. The model's values are scalars, so only the mismatchPathVectors group and the native vector tests reach the rule.", + "typescript": { + "edits": [ + { + "path": "typescript/src/internal/shadow-log-json.ts", + "before": " if (leftItems.length !== rightItems.length) {\n report({ cached: left, source: right });\n return;\n }\n for (let index = 0; index < leftItems.length; index += 1) {", + "after": " for (let index = 0; index < Math.min(leftItems.length, rightItems.length); index += 1) {" + } + ], + "requiredDetections": [ + "ordinary", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/shadow_log.go", + "before": "\t\t\tif len(leftItems) != len(rightItems) {\n\t\t\t\treport(left, right)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tfor index := range leftItems {", + "after": "\t\t\tfor index := range min(len(leftItems), len(rightItems)) {" + } + ], + "requiredDetections": [ + "fixed", + "portable" + ] + } + }, + { + "id": "M67", + "case": "W10.mismatch-paths", + "description": "Report nothing for a pair of values of different JSON kinds, roots included", + "rationale": "TypeScript typescript/src/internal/shadow-log-json.ts walkJson, Go go/shadow_log.go walkJSON and Rust rust/src/protocol/mismatch.rs Walk::walk return without reporting when the two values at a pointer are of different kinds (object, array or primitive). Roots of different kinds therefore yield no entry instead of one entry at \"\" carrying both roots, and a member that changes kind is not reported. Primitives of different JSON types share the primitive kind, so the model's scalars never reach the rule; the mismatchPathVectors group and the native vector tests do.", + "typescript": { + "edits": [ + { + "path": "typescript/src/internal/shadow-log-json.ts", + "before": " if (kind !== jsonKind(right) || kind === \"primitive\") {\n if (left !== right) {", + "after": " if (kind !== jsonKind(right)) {\n return;\n }\n if (kind === \"primitive\") {\n if (left !== right) {" + } + ], + "requiredDetections": [ + "ordinary", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/shadow_log.go", + "before": "\t\tif kind != kindOfJSON(right) {\n\t\t\treport(left, right)\n\t\t\treturn\n\t\t}", + "after": "\t\tif kind != kindOfJSON(right) {\n\t\t\treturn\n\t\t}" + } + ], + "requiredDetections": [ + "fixed", + "portable" + ] + } + }, + { + "id": "M68", + "case": "W10.mismatch-paths", + "description": "Treat an absent compared value as a value with no JSON rendering", + "rationale": "TypeScript typescript/src/internal/shadow-log-json.ts renderShadowMismatchLog counts any side without a rendering as a render failure, the top-level undefined included; Go go/shadow_log.go renderRoot reports an Absent root as failed; Rust rust/src/shadow_log.rs rendered_side maps the absent marker to a side with no rendering. An absent side against a present one therefore logs null difference fields with render_error instead of one entry at \"\" carrying the present side (D13). TypeScript and Go also fail the value fields this way, while Rust's value fields keep their absent handling. The shadow profile has no absent value, so only the absent-side vectors reach the rule.", + "typescript": { + "edits": [ + { + "path": "typescript/src/internal/shadow-log-json.ts", + "before": " && ((cachedJson === null && cachedValue !== undefined) || (sourceJson === null && sourceValue !== undefined));", + "after": " && (cachedJson === null || sourceJson === null);" + } + ], + "requiredDetections": [ + "ordinary", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/shadow_log.go", + "before": "func renderRoot(value any) (encoded []byte, failed bool) {\n\tif IsAbsent(value) {\n\t\treturn nil, false", + "after": "func renderRoot(value any) (encoded []byte, failed bool) {\n\tif IsAbsent(value) {\n\t\treturn nil, true" + } + ], + "requiredDetections": [ + "fixed", + "portable" + ] + } + }, + { + "id": "M69", + "case": "C60.logging-outcomes", + "description": "Fall back to the built-in walk when the mismatch diff hook fails", + "rationale": "TypeScript typescript/src/internal/shadow-log-json.ts renderShadowMismatchLog, Go go/shadow_log.go renderShadowMismatchLog and Rust rust/src/shadow_log.rs render_mismatch_log keep the hook_error outcome when the shadowMismatchDiff hook fails but then walk both renderings, unless a rendering already failed, and log the walk's paths, count and entries. The warning therefore carries difference fields the use case's hook did not produce instead of null ones (D6). Each port renders the sides for the fallback itself, since with a hook and value off the warning renders neither.", + "typescript": { + "edits": [ + { + "path": "typescript/src/internal/shadow-log-json.ts", + "before": " diff = runShadowMismatchHook(hook, cachedValue, sourceValue);\n diffFailed = diff === null;", + "after": " diff = runShadowMismatchHook(hook, cachedValue, sourceValue);\n diffFailed = diff === null;\n if (diff === null && !renderFailed) {\n diff = walkRenderedJson(renderJson(cachedValue), renderJson(sourceValue));\n }" + } + ], + "requiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/shadow_log.go", + "before": "\t\t\tentries, count, computed = runMismatchHook(hook)\n\t\t\tdiffFailed = !computed", + "after": "\t\t\tentries, count, computed = runMismatchHook(hook)\n\t\t\tdiffFailed = !computed\n\t\t\tif !computed && !renderFailed {\n\t\t\t\tcachedRendering, _ := renderRoot(cached)\n\t\t\t\tsourceRendering, _ := renderRoot(source)\n\t\t\t\tentries, count, computed = walkRenderedJSON(cachedRendering, sourceRendering)\n\t\t\t}" + } + ], + "requiredDetections": [ + "generated", + "portable" + ] + } + }, + { + "id": "M70", + "case": "C60.outcome-precedence", + "description": "Run the mismatch diff hook when diff is off", + "rationale": "TypeScript typescript/src/internal/shadow-log-json.ts renderShadowMismatchLog, Go go/shadow_log.go renderShadowMismatchLog and Rust rust/src/shadow_log.rs render_mismatch_log call the shadowMismatchDiff hook when the captured diff leaf is off and record its failure as the warning's failure. A warning that logs only the value fields therefore reports hook_error from a hook that should never have run, outranking the render_error or logged outcome it owes.", + "typescript": { + "edits": [ + { + "path": "typescript/src/internal/shadow-log-json.ts", + "before": " let diffFields: ShadowMismatchLogFields = {};\n if (plan.diff !== \"off\") {", + "after": " let diffFields: ShadowMismatchLogFields = {};\n if (plan.diff === \"off\" && hook !== undefined) {\n diffFailed = runShadowMismatchHook(hook, cachedValue, sourceValue) === null;\n }\n if (plan.diff !== \"off\") {" + } + ], + "requiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/shadow_log.go", + "before": "\tif diff.on() {\n\t\twithValues := diff == MismatchDiffValues", + "after": "\tif !diff.on() && hook != nil {\n\t\t_, _, computed := runMismatchHook(hook)\n\t\tdiffFailed = !computed\n\t}\n\tif diff.on() {\n\t\twithValues := diff == MismatchDiffValues" + } + ], + "requiredDetections": [ + "generated", + "portable" + ] + } + }, + { + "id": "M71", + "case": "C60.shorthand-table", + "description": "Treat a mismatch-logging leaf written off as unwritten", + "rationale": "TypeScript typescript/src/internal/runtime-config.ts expandMismatchLogging, Go go/policy.go expandMismatchLogging and Rust rust/src/policy.rs expand_mismatch_logging ignore a mismatchLogging value leaf written false and a diff leaf written \"off\", as if the group did not write them. A runtime leaf written off therefore falls through to the inherited leaf instead of replacing it, and a leaf written off beside a true logMismatches no longer overrides what the shorthand stands for.", + "typescript": { + "edits": [ + { + "path": "typescript/src/internal/runtime-config.ts", + "before": " const value = written.value !== undefined ? written.value : expanded.value;\n const diff = written.diff !== undefined ? written.diff : expanded.diff;", + "after": " const value = written.value !== undefined && written.value !== false ? written.value : expanded.value;\n const diff = written.diff !== undefined && written.diff !== \"off\" ? written.diff : expanded.diff;" + } + ], + "requiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/policy.go", + "before": "\tif v, present := optionalLeaf(group, \"value\"); present {\n\t\tleaves.value = v\n\t}\n\tif v, present := optionalLeaf(group, \"diff\"); present {\n\t\tleaves.diff = v\n\t}", + "after": "\tif v, present := optionalLeaf(group, \"value\"); present && v != false {\n\t\tleaves.value = v\n\t}\n\tif v, present := optionalLeaf(group, \"diff\"); present && v != \"off\" && v != MismatchDiffOff {\n\t\tleaves.diff = v\n\t}" + } + ], + "requiredDetections": [ + "generated", + "portable" + ] + } + }, + { + "id": "M72", + "case": "C60.outcome-precedence", + "description": "Report render_error over hook_error when both fail", + "rationale": "TypeScript typescript/src/internal/shadow-log-json.ts renderShadowMismatchLog, Go go/shadow_log.go renderShadowMismatchLog and Rust rust/src/shadow_log.rs render_mismatch_log check the render failure before the difference-computation failure when they name the warning's failure. A failing shadowMismatchDiff hook beside a side with no JSON rendering therefore reports render_error, as the warning's loggingOutcome and the shadowMismatchLogging event, instead of hook_error.", + "typescript": { + "edits": [ + { + "path": "typescript/src/internal/shadow-log-json.ts", + "before": " failure: diffFailed ? \"hook_error\" : renderFailed ? \"render_error\" : null,", + "after": " failure: renderFailed ? \"render_error\" : diffFailed ? \"hook_error\" : null," + } + ], + "requiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/shadow_log.go", + "before": "\tcase diffFailed:\n\t\tfailure = shadowLogHookError\n\tcase renderFailed:\n\t\tfailure = shadowLogRenderError", + "after": "\tcase renderFailed:\n\t\tfailure = shadowLogRenderError\n\tcase diffFailed:\n\t\tfailure = shadowLogHookError" + } + ], + "requiredDetections": [ + "generated", + "portable" + ] + } + }, + { + "id": "M73", + "case": "C60.outcome-precedence", + "description": "Report logger_error over a warning construction failure when the logger also fails", + "rationale": "TypeScript typescript/src/dialcache.ts warnShadowMismatch, Go go/shadow.go warnShadowMismatch and Rust rust/src/shadow.rs log_shadow_mismatch let a failed logger decide the shadowMismatchLogging outcome before the construction failure the warning names. A warning whose hook failed or whose side had no rendering therefore records logger_error when the logger also fails, instead of hook_error or render_error; the warning's own loggingOutcome field is unchanged.", + "typescript": { + "edits": [ + { + "path": "typescript/src/dialcache.ts", + "before": " if (failure !== null) {\n recordOutcome(failure);\n } else if (typeof delivered === \"boolean\") {", + "after": " if (failure !== null && delivered !== false) {\n recordOutcome(failure);\n } else if (typeof delivered === \"boolean\") {" + } + ], + "requiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/shadow.go", + "before": "\toutcome := failure\n\tif outcome == \"\" {\n\t\toutcome = \"logged\"\n\t\tif !delivered {\n\t\t\toutcome = \"logger_error\"\n\t\t}\n\t}", + "after": "\toutcome := failure\n\tif !delivered {\n\t\toutcome = \"logger_error\"\n\t} else if outcome == \"\" {\n\t\toutcome = \"logged\"\n\t}" + } + ], + "requiredDetections": [ + "generated", + "portable" + ] + } + }, + { + "id": "M74", + "case": "C60.logging-outcomes", + "description": "Emit a mismatch warning when both mismatch-logging leaves are off", + "rationale": "TypeScript typescript/src/dialcache.ts resolveShadowLogging returns a log plan whatever the leaves, Go go/policy.go ResolvedShadow.logsMismatches and Rust rust/src/policy.rs ResolvedShadow::logs_mismatches report true. Every confirmed mismatch therefore emits a metadata-only warning and a logged shadowMismatchLogging event, even when value is false and diff is off, instead of nothing.", + "typescript": { + "edits": [ + { + "path": "typescript/src/dialcache.ts", + "before": " return value || diff !== \"off\" ? { value, diff } : null;", + "after": " return { value, diff };" + } + ], + "requiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/policy.go", + "before": "func (s ResolvedShadow) logsMismatches() bool { return s.LogValue || s.LogDiff.on() }", + "after": "func (s ResolvedShadow) logsMismatches() bool { return true }" + } + ], + "requiredDetections": [ + "generated", + "portable" + ] + } + }, + { + "id": "M75", + "case": "C60.warning-field-groups", + "description": "Log the difference entries with their values at diff paths", + "rationale": "TypeScript typescript/src/internal/shadow-log-json.ts renderShadowMismatchLog, Go go/shadow_log.go renderShadowMismatchLog and Rust rust/src/shadow_log.rs render_mismatch_log treat every diff level above off as values, so a warning at diff paths also carries diffJson with each entry's cached and source sides. The paths level, which a cache of user data can leave on in production, therefore logs values.", + "typescript": { + "edits": [ + { + "path": "typescript/src/internal/shadow-log-json.ts", + "before": " const withValues = plan.diff === \"values\";", + "after": " const withValues = plan.diff === \"values\" || plan.diff === \"paths\";" + } + ], + "requiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/shadow_log.go", + "before": "\t\twithValues := diff == MismatchDiffValues", + "after": "\t\twithValues := diff.on()" + } + ], + "requiredDetections": [ + "generated", + "portable" + ] + } + }, + { + "id": "M76", + "case": "C60.invalid-input-isolation", + "description": "Resolve an invalid diff leaf to values instead of off", + "rationale": "TypeScript typescript/src/internal/runtime-config.ts resolveMismatchLogging, Go go/policy.go resolveDiffLeaf and Rust rust/src/policy.rs resolve_mismatch_logging resolve a diff leaf of the wrong type or an unknown level to values while still recording its config_resolution. An invalid diff leaf therefore turns on the most exposing level instead of acting off, beside a valid value leaf or alone. Go's static ParsePolicy rejects an invalid level before resolution, so only a runtime overlay reaches the edit.", + "typescript": { + "edits": [ + { + "path": "typescript/src/internal/runtime-config.ts", + "before": " diff: diff ?? \"off\",", + "after": " diff: diff ?? \"values\"," + } + ], + "requiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + "go": { + "edits": [ + { + "path": "go/policy.go", + "before": "\tif !level.valid() {\n\t\treturn MismatchDiffOff, false", + "after": "\tif !level.valid() {\n\t\treturn MismatchDiffValues, false" + } + ], + "requiredDetections": [ + "generated", + "portable" + ] + } } ] } diff --git a/formal/catalogs/profile-lint-baseline.json b/formal/catalogs/profile-lint-baseline.json index 571db1ac..9acea567 100644 --- a/formal/catalogs/profile-lint-baseline.json +++ b/formal/catalogs/profile-lint-baseline.json @@ -18,6 +18,7 @@ "local_storage", "markers", "metrics", + "mismatch_logging", "payloads", "policy_gate", "policy_overlay", @@ -120,6 +121,8 @@ "module": "dialcache_shadow_conformance", "libraryTransitions": [ "clock::shiftWall", + "mismatch_logging::mergeLogging", + "mismatch_logging::resolveLogging", "remote_frames::seedFrame", "remote_io::latestOrdinal", "remote_writes::latestDump", @@ -127,11 +130,14 @@ "serving::invalidate", "shadow::advanceHeld", "shadow::beginDark", + "shadow::decoderOf", "shadow::latestLoad", "shadow::latestRead", + "shadow::readerOf", + "shadow::recordVerdictInputs", "shadow::releaseDump", "shadow::releaseLoad", - "shadow::releaseRead", + "shadow::releaseReadWithLogger", "shadow::releaseWrite", "shadow::settleHeld" ], diff --git a/formal/catalogs/profiles.json b/formal/catalogs/profiles.json index 663614f3..e31ffb29 100644 --- a/formal/catalogs/profiles.json +++ b/formal/catalogs/profiles.json @@ -94,7 +94,7 @@ }, { "id": "shadow", - "version": 3, + "version": 4, "definition": "formal/guides/BEHAVIOR.md", "model": "formal/models/dialcache-shadow-conformance.qnt", "smoke": "formal/generated/shadow-smoke.itf.json", diff --git a/formal/catalogs/protocol-vectors.json b/formal/catalogs/protocol-vectors.json index b66b3633..290ff53b 100644 --- a/formal/catalogs/protocol-vectors.json +++ b/formal/catalogs/protocol-vectors.json @@ -1464,5 +1464,64 @@ "args": [] } } + ], + "mismatchPathVectors": [ + {"name":"identical objects","cached":{"id":"123","tags":["a","b"],"meta":{},"list":[],"profile":{"email":"a@example.com"}},"source":{"id":"123","tags":["a","b"],"meta":{},"list":[],"profile":{"email":"a@example.com"}},"entries":[],"count":0}, + {"name":"identical primitives","cached":"same","source":"same","entries":[],"count":0}, + {"name":"identical objects with different member order","cached":{"a":1,"b":{"c":2,"d":3}},"source":{"b":{"d":3,"c":2},"a":1},"entries":[],"count":0}, + {"name":"differing number roots","cached":1,"source":2,"entries":[{"path":"","cached":1,"source":2}],"count":1}, + {"name":"differing string roots","cached":"old","source":"new","entries":[{"path":"","cached":"old","source":"new"}],"count":1}, + {"name":"null root against false root","cached":null,"source":false,"entries":[{"path":"","cached":null,"source":false}],"count":1}, + {"name":"number root against string root","cached":1,"source":"1","entries":[{"path":"","cached":1,"source":"1"}],"count":1}, + {"name":"object root against array root","cached":{},"source":[],"entries":[{"path":"","cached":{},"source":[]}],"count":1}, + {"name":"array root against string root","cached":["a"],"source":"a","entries":[{"path":"","cached":["a"],"source":"a"}],"count":1}, + {"name":"null root against object root","cached":null,"source":{"a":null},"entries":[{"path":"","cached":null,"source":{"a":null}}],"count":1}, + {"name":"nested member change","cached":{"id":"123","profile":{"email":"old@example.com","name":"Ada"}},"source":{"id":"123","profile":{"email":"new@example.com","name":"Ada"}},"entries":[{"path":"/profile/email","cached":"old@example.com","source":"new@example.com"}],"count":1}, + {"name":"nested members changing kind","cached":{"a":[1],"b":{"c":1},"d":null},"source":{"a":{"0":1},"b":"c","d":{}},"entries":[{"path":"/a","cached":[1],"source":{"0":1}},{"path":"/b","cached":{"c":1},"source":"c"},{"path":"/d","cached":null,"source":{}}],"count":3}, + {"name":"member only in cached","cached":{"id":1,"legacy":{"flag":true}},"source":{"id":1},"entries":[{"path":"/legacy","cached":{"flag":true}}],"count":1}, + {"name":"member only in source","cached":{"id":1},"source":{"id":1,"roles":["admin"]},"entries":[{"path":"/roles","source":["admin"]}],"count":1}, + {"name":"one-sided members on both sides","cached":{"a":1,"b":2},"source":{"b":2,"c":3},"entries":[{"path":"/a","cached":1},{"path":"/c","source":3}],"count":2}, + {"name":"one-sided null member","cached":{"a":null},"source":{},"entries":[{"path":"/a","cached":null}],"count":1}, + {"name":"member names in UTF-16 code-unit order","cached":{"～":1,"b":1,"😀":1,"a":1,"B":1},"source":{"a":2,"😀":2,"B":2,"～":2,"b":2},"entries":[{"path":"/B","cached":1,"source":2},{"path":"/a","cached":1,"source":2},{"path":"/b","cached":1,"source":2},{"path":"/😀","cached":1,"source":2},{"path":"/～","cached":1,"source":2}],"count":5}, + {"name":"integer-like member names in string order","cached":{"9":1,"10":1,"a":1},"source":{"9":2,"10":2,"a":2},"entries":[{"path":"/10","cached":1,"source":2},{"path":"/9","cached":1,"source":2},{"path":"/a","cached":1,"source":2}],"count":3}, + {"name":"equal-length arrays recursed by index","cached":{"items":[1,2,3]},"source":{"items":[1,5,4]},"entries":[{"path":"/items/1","cached":2,"source":5},{"path":"/items/2","cached":3,"source":4}],"count":2}, + {"name":"equal-length arrays of objects recursed into each item","cached":[{"id":1,"name":"a"},{"id":2,"name":"b"}],"source":[{"id":1,"name":"a"},{"id":2,"name":"c"}],"entries":[{"path":"/1/name","cached":"b","source":"c"}],"count":1}, + {"name":"different-length root arrays reported once","cached":[1,2],"source":[1,2,3],"entries":[{"path":"","cached":[1,2],"source":[1,2,3]}],"count":1}, + {"name":"different-length nested arrays reported once at the array pointer","cached":{"name":"x","roles":["admin"],"tags":[]},"source":{"name":"y","roles":["admin","owner"],"tags":[null]},"entries":[{"path":"/name","cached":"x","source":"y"},{"path":"/roles","cached":["admin"],"source":["admin","owner"]},{"path":"/tags","cached":[],"source":[null]}],"count":3}, + {"name":"escaped reference tokens and the empty member name","cached":{"m~n":1,"a/b":1,"/":1,"~1":1,"":1},"source":{"":2,"~1":2,"/":2,"a/b":2,"m~n":2},"entries":[{"path":"/","cached":1,"source":2},{"path":"/~1","cached":1,"source":2},{"path":"/a~1b","cached":1,"source":2},{"path":"/m~0n","cached":1,"source":2},{"path":"/~01","cached":1,"source":2}],"count":5}, + {"name":"nested empty member names","cached":{"":{"":"x"}},"source":{"":{"":"y"}},"entries":[{"path":"//","cached":"x","source":"y"}],"count":1}, + {"name":"escaped tokens composed with array indices","cached":{"a/b":[0,{"~":1}]},"source":{"a/b":[0,{"~":2}]},"entries":[{"path":"/a~1b/1/~0","cached":1,"source":2}],"count":1}, + {"name":"entries in walk order","cached":{"z":[1,{"b":1,"a":1}],"a":{"y":1,"x":1}},"source":{"a":{"x":2,"y":2},"z":[2,{"a":2,"b":2}]},"entries":[{"path":"/a/x","cached":1,"source":2},{"path":"/a/y","cached":1,"source":2},{"path":"/z/0","cached":1,"source":2},{"path":"/z/1/a","cached":1,"source":2},{"path":"/z/1/b","cached":1,"source":2}],"count":5}, + {"name":"difference at the depth cap reported at its own pointer","cached":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],"source":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[2]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],"entries":[{"path":"/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0","cached":1,"source":2}],"count":1}, + {"name":"difference past the depth cap reported at the 32-token container","cached":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],"source":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[2]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],"entries":[{"path":"/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0","cached":[1],"source":[2]}],"count":1}, + {"name":"equal containers past the depth cap with different member order","cached":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[{"a":1,"b":{"c":[1,2],"d":null}}]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],"source":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[{"b":{"d":null,"c":[1,2]},"a":1}]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],"entries":[],"count":0}, + {"name":"containers at the depth cap reported whole","cached":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[{"kinds":{"m":[1]},"kindsSwap":{"m":{"0":1}},"lengths":[[1,2]],"lengthsShort":[[1]],"members":{"k":1},"order":{"x":1,"y":{"p":null,"q":[true]}},"sizes":{"a":1},"values":[1]}]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],"source":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[{"values":[2],"sizes":{"a":1,"b":2},"order":{"y":{"q":[true],"p":null},"x":1},"members":{"j":1},"lengthsShort":[[1,2]],"lengths":[[1]],"kindsSwap":{"m":[1]},"kinds":{"m":{"0":1}}}]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]],"entries":[{"path":"/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/kinds","cached":{"m":[1]},"source":{"m":{"0":1}}},{"path":"/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/kindsSwap","cached":{"m":{"0":1}},"source":{"m":[1]}},{"path":"/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/lengths","cached":[[1,2]],"source":[[1]]},{"path":"/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/lengthsShort","cached":[[1]],"source":[[1,2]]},{"path":"/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/members","cached":{"k":1},"source":{"j":1}},{"path":"/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/sizes","cached":{"a":1},"source":{"a":1,"b":2}},{"path":"/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/values","cached":[1],"source":[2]}],"count":7}, + {"name":"documented example: a user record at four differing fields","cached":{"id":"u_123","email":"ana@example.com","plan":"free","roles":["viewer"],"profile":{"name":"Ana","locale":"en-US"}},"source":{"id":"u_123","email":"ana@example.com","plan":"pro","roles":["viewer","editor"],"profile":{"name":"Ana","locale":"en-GB","timezone":"Europe/London"}},"entries":[{"path":"/plan","cached":"free","source":"pro"},{"path":"/profile/locale","cached":"en-US","source":"en-GB"},{"path":"/profile/timezone","source":"Europe/London"},{"path":"/roles","cached":["viewer"],"source":["viewer","editor"]}],"count":4}, + {"name":"documented example: a changed field","cached":{"plan":"free"},"source":{"plan":"pro"},"entries":[{"path":"/plan","cached":"free","source":"pro"}],"count":1}, + {"name":"documented example: an element changed in place","cached":{"tags":["a","b"]},"source":{"tags":["a","c"]},"entries":[{"path":"/tags/1","cached":"b","source":"c"}],"count":1}, + {"name":"documented example: an element inserted","cached":{"tags":["a","b"]},"source":{"tags":["a","x","b"]},"entries":[{"path":"/tags","cached":["a","b"],"source":["a","x","b"]}],"count":1}, + {"name":"documented example: a field only the source has","cached":{"name":"Ana"},"source":{"name":"Ana","nickname":"A"},"entries":[{"path":"/nickname","source":"A"}],"count":1}, + {"name":"documented example: a field only the cache has","cached":{"name":"Ana","nickname":"A"},"source":{"name":"Ana"},"entries":[{"path":"/nickname","cached":"A"}],"count":1}, + {"name":"documented example: a field changing type","cached":{"count":1},"source":{"count":"1"},"entries":[{"path":"/count","cached":1,"source":"1"}],"count":1}, + {"name":"documented example: an object replaced by null","cached":{"address":{"city":"Paris"}},"source":{"address":null},"entries":[{"path":"/address","cached":{"city":"Paris"},"source":null}],"count":1}, + {"name":"documented example: member names with escaped characters","cached":{"a/b":1,"m~n":1},"source":{"a/b":2,"m~n":2},"entries":[{"path":"/a~1b","cached":1,"source":2},{"path":"/m~0n","cached":1,"source":2}],"count":2}, + {"name":"documented example: roots of different kinds","cached":["a","b"],"source":{"items":["a","b"]},"entries":[{"path":"","cached":["a","b"],"source":{"items":["a","b"]}}],"count":1}, + {"name":"own members named like Object.prototype members","cached":{"constructor":1,"__proto__":{"a":1}},"source":{"toString":"x","valueOf":null,"__proto__":{"a":2}},"entries":[{"path":"/__proto__/a","cached":1,"source":2},{"path":"/constructor","cached":1},{"path":"/toString","source":"x"},{"path":"/valueOf","source":null}],"count":4}, + {"name":"own members only in a depth-capped pair","cached":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"__proto__":{}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}},"source":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"b":{}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}},"entries":[{"path":"/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a","cached":{"__proto__":{}},"source":{"b":{}}}],"count":1}, + {"name":"entries cut at 64 in walk order with the uncut count","cached":{"m69":0,"m68":0,"m67":0,"m66":0,"m65":0,"m64":0,"m63":0,"m62":0,"m61":0,"m60":0,"m59":0,"m58":0,"m57":0,"m56":0,"m55":0,"m54":0,"m53":0,"m52":0,"m51":0,"m50":0,"m49":0,"m48":0,"m47":0,"m46":0,"m45":0,"m44":0,"m43":0,"m42":0,"m41":0,"m40":0,"m39":0,"m38":0,"m37":0,"m36":0,"m35":0,"m34":0,"m33":0,"m32":0,"m31":0,"m30":0,"m29":0,"m28":0,"m27":0,"m26":0,"m25":0,"m24":0,"m23":0,"m22":0,"m21":0,"m20":0,"m19":0,"m18":0,"m17":0,"m16":0,"m15":0,"m14":0,"m13":0,"m12":0,"m11":0,"m10":0,"m09":0,"m08":0,"m07":0,"m06":0,"m05":0,"m04":0,"m03":0,"m02":0,"m01":0,"m00":0},"source":{"m69":1,"m68":1,"m67":1,"m66":1,"m65":1,"m64":1,"m63":1,"m62":1,"m61":1,"m60":1,"m59":1,"m58":1,"m57":1,"m56":1,"m55":1,"m54":1,"m53":1,"m52":1,"m51":1,"m50":1,"m49":1,"m48":1,"m47":1,"m46":1,"m45":1,"m44":1,"m43":1,"m42":1,"m41":1,"m40":1,"m39":1,"m38":1,"m37":1,"m36":1,"m35":1,"m34":1,"m33":1,"m32":1,"m31":1,"m30":1,"m29":1,"m28":1,"m27":1,"m26":1,"m25":1,"m24":1,"m23":1,"m22":1,"m21":1,"m20":1,"m19":1,"m18":1,"m17":1,"m16":1,"m15":1,"m14":1,"m13":1,"m12":1,"m11":1,"m10":1,"m09":1,"m08":1,"m07":1,"m06":1,"m05":1,"m04":1,"m03":1,"m02":1,"m01":1,"m00":1},"entries":[{"path":"/m00","cached":0,"source":1},{"path":"/m01","cached":0,"source":1},{"path":"/m02","cached":0,"source":1},{"path":"/m03","cached":0,"source":1},{"path":"/m04","cached":0,"source":1},{"path":"/m05","cached":0,"source":1},{"path":"/m06","cached":0,"source":1},{"path":"/m07","cached":0,"source":1},{"path":"/m08","cached":0,"source":1},{"path":"/m09","cached":0,"source":1},{"path":"/m10","cached":0,"source":1},{"path":"/m11","cached":0,"source":1},{"path":"/m12","cached":0,"source":1},{"path":"/m13","cached":0,"source":1},{"path":"/m14","cached":0,"source":1},{"path":"/m15","cached":0,"source":1},{"path":"/m16","cached":0,"source":1},{"path":"/m17","cached":0,"source":1},{"path":"/m18","cached":0,"source":1},{"path":"/m19","cached":0,"source":1},{"path":"/m20","cached":0,"source":1},{"path":"/m21","cached":0,"source":1},{"path":"/m22","cached":0,"source":1},{"path":"/m23","cached":0,"source":1},{"path":"/m24","cached":0,"source":1},{"path":"/m25","cached":0,"source":1},{"path":"/m26","cached":0,"source":1},{"path":"/m27","cached":0,"source":1},{"path":"/m28","cached":0,"source":1},{"path":"/m29","cached":0,"source":1},{"path":"/m30","cached":0,"source":1},{"path":"/m31","cached":0,"source":1},{"path":"/m32","cached":0,"source":1},{"path":"/m33","cached":0,"source":1},{"path":"/m34","cached":0,"source":1},{"path":"/m35","cached":0,"source":1},{"path":"/m36","cached":0,"source":1},{"path":"/m37","cached":0,"source":1},{"path":"/m38","cached":0,"source":1},{"path":"/m39","cached":0,"source":1},{"path":"/m40","cached":0,"source":1},{"path":"/m41","cached":0,"source":1},{"path":"/m42","cached":0,"source":1},{"path":"/m43","cached":0,"source":1},{"path":"/m44","cached":0,"source":1},{"path":"/m45","cached":0,"source":1},{"path":"/m46","cached":0,"source":1},{"path":"/m47","cached":0,"source":1},{"path":"/m48","cached":0,"source":1},{"path":"/m49","cached":0,"source":1},{"path":"/m50","cached":0,"source":1},{"path":"/m51","cached":0,"source":1},{"path":"/m52","cached":0,"source":1},{"path":"/m53","cached":0,"source":1},{"path":"/m54","cached":0,"source":1},{"path":"/m55","cached":0,"source":1},{"path":"/m56","cached":0,"source":1},{"path":"/m57","cached":0,"source":1},{"path":"/m58","cached":0,"source":1},{"path":"/m59","cached":0,"source":1},{"path":"/m60","cached":0,"source":1},{"path":"/m61","cached":0,"source":1},{"path":"/m62","cached":0,"source":1},{"path":"/m63","cached":0,"source":1}],"count":70}, + {"name":"pointers clamped to 512 bytes on a UTF-8 boundary","cached":{"kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk":1,"kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk":1,"éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé":1},"source":{"kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk":2,"kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk":2,"éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé":2},"entries":[{"path":"/kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk","cached":1,"source":2},{"path":"/kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk...[truncated]","cached":1,"source":2},{"path":"/éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé...[truncated]","cached":1,"source":2}],"count":3}, + {"name":"each side over 1 KiB replaced by its clamped JSON text","cached":{"accent":"éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé","blob":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","edge":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","extra":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz","list":[1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000]},"source":{"accent":"a","blob":"y","edge":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","list":[1000]},"entries":[{"path":"/accent","cached":"\"éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé...[truncated]","source":"a"},{"path":"/blob","cached":"\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...[truncated]","source":"y"},{"path":"/edge","cached":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","source":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},{"path":"/extra","cached":"\"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...[truncated]"},{"path":"/list","cached":"[1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000...[truncated]","source":[1000]}],"count":5}, + {"name":"difference below the depth cap reported at the 32-token container","cached":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}},"source":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":2}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}},"entries":[{"path":"/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n/n","cached":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":1}}}}}}}},"source":{"n":{"n":{"n":{"n":{"n":{"n":{"n":{"n":2}}}}}}}}}],"count":1}, + {"name":"absent cached side against a present source","source":{"plan":"pro"},"entries":[{"path":"","source":{"plan":"pro"}}],"count":1}, + {"name":"present cached side against an absent source","cached":{"version":2},"entries":[{"path":"","cached":{"version":2}}],"count":1}, + {"name":"absent cached side against a null source","source":null,"entries":[{"path":"","source":null}],"count":1}, + {"name":"two absent sides","entries":[],"count":0}, + {"name":"numbers compare as doubles: 1 and 1.0","cached":{"n":1},"source":{"n":1.0},"entries":[],"count":0}, + {"name":"a bool never equals a number: true and 1","cached":true,"source":1,"entries":[{"path":"","cached":true,"source":1}],"count":1}, + {"name":"a bool never equals a number: false and 0","cached":false,"source":0,"entries":[{"path":"","cached":false,"source":0}],"count":1}, + {"name":"numbers compare as doubles: 0 and -0","cached":0,"source":-0,"entries":[],"count":0}, + {"name":"numbers compare as doubles: 2^53 + 1 and 2^53","cached":{"id":9007199254740993},"source":{"id":9007199254740992},"entries":[],"count":0}, + {"name":"pointer of exactly 512 bytes above a differing member","cached":{"kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk":{"a":1}},"source":{"kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk":{"a":2}},"entries":[{"path":"/kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk...[truncated]","cached":1,"source":2}],"count":1}, + {"name":"pointer of exactly 512 bytes above a differing array element","cached":{"kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk":[1]},"source":{"kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk":[2]},"entries":[{"path":"/kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk...[truncated]","cached":1,"source":2}],"count":1} ] } diff --git a/formal/catalogs/rust-mutations.json b/formal/catalogs/rust-mutations.json index 89909ac0..20e58b52 100644 --- a/formal/catalogs/rust-mutations.json +++ b/formal/catalogs/rust-mutations.json @@ -170,8 +170,8 @@ "edits": [ { "path": "rust/src/policy.rs", - "before": " shadow.log_mismatches = false;\n shadow.logging_config_error = true;", - "after": " shadow.log_mismatches = true;\n shadow.logging_config_error = true;" + "before": " Some(_) => expanded(false, MismatchDiff::Off, true),", + "after": " Some(_) => expanded(true, MismatchDiff::Values, true)," } ], "requiredDetections": [ @@ -193,8 +193,8 @@ "edits": [ { "path": "rust/src/policy.rs", - "before": " let mut shadow = ResolvedShadow::default();", - "after": " let mut shadow = ResolvedShadow {\n log_mismatches: true,\n ..ResolvedShadow::default()\n };" + "before": " let value = match &leaves.value {\n None => Some(false),", + "after": " let value = match &leaves.value {\n None => Some(true)," } ], "requiredDetections": [ @@ -297,6 +297,270 @@ "generated", "portable" ] + }, + { + "id": "M14", + "case": "C60.logging-enabled", + "description": "Leave mismatch logging off when its flag is true", + "typescriptMutation": "M65", + "edits": [ + { + "path": "rust/src/policy.rs", + "before": " Some(Value::Bool(true)) => expanded(true, MismatchDiff::Values, false),", + "after": " Some(Value::Bool(true)) => expanded(false, MismatchDiff::Off, false)," + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M15", + "case": "W10.mismatch-paths", + "description": "Walk arrays of different length over their common prefix instead of reporting them once at the array pointer", + "typescriptMutation": "M66", + "edits": [ + { + "path": "rust/src/protocol/mismatch.rs", + "before": " if left_items.len() != right_items.len() {\n self.report(Some(left), Some(right));\n return;\n }\n for (index, (l, r)) in left_items.iter().zip(right_items).enumerate() {", + "after": " for (index, (l, r)) in left_items.iter().zip(right_items).enumerate() {" + } + ], + "requiredDetections": [ + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "portable" + ] + }, + { + "id": "M16", + "case": "W10.mismatch-paths", + "description": "Report nothing for a pair of values of different JSON kinds, roots included", + "typescriptMutation": "M67", + "edits": [ + { + "path": "rust/src/protocol/mismatch.rs", + "before": " if kind != Kind::of(right) || kind == Kind::Primitive {\n if !primitives_equal(left, right) {", + "after": " if kind != Kind::of(right) {\n return;\n }\n if kind == Kind::Primitive {\n if !primitives_equal(left, right) {" + } + ], + "requiredDetections": [ + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "portable" + ] + }, + { + "id": "M17", + "case": "W10.mismatch-paths", + "description": "Treat an absent compared value as a value with no JSON rendering", + "typescriptMutation": "M68", + "edits": [ + { + "path": "rust/src/shadow_log.rs", + "before": " Some(Rendering::Absent) => Some(None),", + "after": " Some(Rendering::Absent) => None," + } + ], + "requiredDetections": [ + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "portable" + ] + }, + { + "id": "M18", + "case": "C60.logging-outcomes", + "description": "Fall back to the built-in walk when the mismatch diff hook fails", + "typescriptMutation": "M69", + "edits": [ + { + "path": "rust/src/shadow_log.rs", + "before": " let entries = run_hook(hook, cached, source, with_values);\n diff_failed = entries.is_none();\n entries\n", + "after": " let entries = run_hook(hook, cached, source, with_values);\n diff_failed = entries.is_none();\n entries.or_else(|| {\n let json = json.filter(|_| !render_failed)?;\n let side = |value: &StoredValue| match json.full(value) {\n Rendering::Json(text) => Some(Some(text)),\n Rendering::Absent => Some(None),\n Rendering::Failed => None,\n };\n let (cached, source) = (side(cached)?, side(source)?);\n diff_rendered(cached.as_deref(), source.as_deref(), with_values).ok()\n })\n" + } + ], + "requiredDetections": [ + "generated", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M19", + "case": "C60.outcome-precedence", + "description": "Run the mismatch diff hook when diff is off", + "typescriptMutation": "M70", + "edits": [ + { + "path": "rust/src/shadow_log.rs", + "before": " let diff = (plan.diff != MismatchDiff::Off).then(|| {", + "after": " if let (MismatchDiff::Off, Some(hook)) = (plan.diff, &renderers.hook) {\n diff_failed = run_hook(hook, cached, source, false).is_none();\n }\n let diff = (plan.diff != MismatchDiff::Off).then(|| {" + } + ], + "requiredDetections": [ + "generated", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M20", + "case": "C60.shorthand-table", + "description": "Treat a mismatch-logging leaf written off as unwritten", + "typescriptMutation": "M71", + "edits": [ + { + "path": "rust/src/policy.rs", + "before": " if let Some(value) = group.get(\"value\") {\n leaves.value = Some(value.clone());\n }\n if let Some(diff) = group.get(\"diff\") {\n leaves.diff = Some(diff.clone());\n }", + "after": " if let Some(value) = group.get(\"value\").filter(|value| **value != Value::Bool(false)) {\n leaves.value = Some(value.clone());\n }\n if let Some(diff) = group.get(\"diff\").filter(|diff| diff.as_str() != Some(\"off\")) {\n leaves.diff = Some(diff.clone());\n }" + } + ], + "requiredDetections": [ + "generated", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M21", + "case": "C60.outcome-precedence", + "description": "Report render_error over hook_error when both fail", + "typescriptMutation": "M72", + "edits": [ + { + "path": "rust/src/shadow_log.rs", + "before": " failure: if diff_failed {\n Some(ShadowMismatchLoggingOutcome::HookError)\n } else if render_failed {\n Some(ShadowMismatchLoggingOutcome::RenderError)\n }", + "after": " failure: if render_failed {\n Some(ShadowMismatchLoggingOutcome::RenderError)\n } else if diff_failed {\n Some(ShadowMismatchLoggingOutcome::HookError)\n }" + } + ], + "requiredDetections": [ + "generated", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M22", + "case": "C60.outcome-precedence", + "description": "Report logger_error over a warning construction failure when the logger also fails", + "typescriptMutation": "M73", + "edits": [ + { + "path": "rust/src/shadow.rs", + "before": " let outcome = content.failure.unwrap_or(if delivered {", + "after": " let outcome = (if delivered { content.failure } else { None }).unwrap_or(if delivered {" + } + ], + "requiredDetections": [ + "generated", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M23", + "case": "C60.logging-outcomes", + "description": "Emit a mismatch warning when both mismatch-logging leaves are off", + "typescriptMutation": "M74", + "edits": [ + { + "path": "rust/src/policy.rs", + "before": " self.mismatch_value || self.mismatch_diff != MismatchDiff::Off\n }", + "after": " true\n }" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M24", + "case": "C60.invalid-input-isolation", + "description": "Resolve an invalid diff leaf to values instead of off", + "typescriptMutation": "M76", + "edits": [ + { + "path": "rust/src/policy.rs", + "before": " diff: diff.unwrap_or(MismatchDiff::Off),", + "after": " diff: diff.unwrap_or(MismatchDiff::Values)," + } + ], + "requiredDetections": [ + "generated", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M25", + "case": "C60.warning-field-groups", + "description": "Log the difference entries with their values at diff paths", + "typescriptMutation": "M75", + "edits": [ + { + "path": "rust/src/shadow_log.rs", + "before": " let with_values = plan.diff == MismatchDiff::Values;", + "after": " let with_values = plan.diff != MismatchDiff::Off;" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] } ] } diff --git a/formal/catalogs/semantic-cases.json b/formal/catalogs/semantic-cases.json index 440499ac..2e43b55a 100644 --- a/formal/catalogs/semantic-cases.json +++ b/formal/catalogs/semantic-cases.json @@ -680,11 +680,11 @@ {"ref": "formal/models/dialcache-effects-conformance.qnt:acquiredDecodeSurvivesInvalidationAndDeadlineTest", "scope": "Remote get duration includes delayed fresh decode while accepted decoding starts no source budget."}]}, {"id": "C59.source-excludes-publication", "rule": "accepted publication time is separate from reported source duration", "scenarios": ["accepted publication time is separate from reported source duration"], "generated": [{"profile": "effects", "witness": "duration:dump"}], "vectors": [], "contracts": ["C59"], "models": [ {"ref": "formal/models/dialcache-effects-conformance.qnt:sourceDurationExcludesPublicationTest", "scope": "Source duration stays at its accepted settlement interval while serialization has a separate later duration."}]}, - {"id": "C60.logging-default-off", "rule": "Omitting the logging flag leaves mismatch warnings disabled", "scenarios": [], "generated": [{"profile": "shadow", "witness": "omitted-logging-defaults-off"}, {"profile": "shadow", "witness": "omitted-shadow-logging-keeps-mismatch-metrics-only"}], "vectors": [], "contracts": ["C60"], "quintReplays": ["shadow/omittedLoggingKeepsMismatchMetricsOnlyTest"], "models": [ + {"id": "C60.logging-default-off", "rule": "Omitting every mismatch-logging input leaves mismatch warnings disabled", "scenarios": [], "generated": [{"profile": "shadow", "witness": "omitted-logging-defaults-off"}, {"profile": "shadow", "witness": "omitted-shadow-logging-keeps-mismatch-metrics-only"}], "vectors": [], "contracts": ["C60"], "quintReplays": ["shadow/omittedLoggingKeepsMismatchMetricsOnlyTest"], "models": [ {"ref": "formal/models/dialcache-shadow-conformance.qnt:omittedLoggingKeepsMismatchMetricsOnlyTest", "scope": "The comparator fixture omits logging entirely. Confirmed mismatch still emits one age sample and verdict, with zero warning and zero configuration failure; public action history never includes a false logging reply."}], "definitions": [ - {"ref": "formal/models/kernel/shadow.qnt:admitDark", "scope": "Omitted-logging fixture starts false, admission captures it, and mismatch warning emission consults the captured flag."}, - {"ref": "formal/models/kernel/shadow.qnt:beginDark", "scope": "Omitted-logging fixture starts false, admission captures it, and mismatch warning emission consults the captured flag."}, - {"ref": "formal/models/kernel/shadow.qnt:recordVerdict", "scope": "Omitted-logging fixture starts false, admission captures it, and mismatch warning emission consults the captured flag."}, + {"ref": "formal/models/kernel/shadow.qnt:admitDark", "scope": "The omitted-logging fixture writes no input, admission resolves and captures both leaves off, and mismatch warning emission consults the captured leaves."}, + {"ref": "formal/models/kernel/shadow.qnt:beginDark", "scope": "The omitted-logging fixture writes no input, admission resolves and captures both leaves off, and mismatch warning emission consults the captured leaves."}, + {"ref": "formal/models/kernel/shadow.qnt:recordVerdict", "scope": "The omitted-logging fixture writes no input, admission resolves and captures both leaves off, and mismatch warning emission consults the captured leaves."}, {"ref": "formal/models/dialcache-shadow-conformance.qnt:initialize", "scope": "The comparator fixture omits logging entirely. Confirmed mismatch still emits one age sample and verdict, with zero warning and zero configuration failure; public action history never includes a false logging reply."}]}, {"id": "C60.logging-enabled", "rule": "mismatch warning enabled with mismatch verdict", "scenarios": ["mismatch warning enabled with mismatch verdict"], "generated": [{"profile": "shadow", "witness": "mismatch-logging:true"}], "vectors": [], "contracts": ["C60"], "models": [ {"ref": "formal/models/dialcache-shadow-conformance.qnt:explicitInequalityRequiresConfirmationTest", "scope": "Confirmed mismatch warns only after C1; accepted enabled logging remains in effect despite later policy disable."}, @@ -696,8 +696,8 @@ {"ref": "formal/models/dialcache-shadow-conformance.qnt:warningsRequireMismatch", "scope": "Warnings never exceed the confirmed mismatches, so a supersession adds none."}], "definitions": [ {"ref": "formal/models/kernel/shadow.qnt:recordVerdict", "scope": "A supersession is neither a match nor a confirmed mismatch: no age sample and no warning."}]}, {"id": "C60.invalid-logging-policy", "rule": "Malformed runtime mismatch logging records configuration failure and leaves an eligible job active with logging off", "scenarios": ["malformed runtime mismatch logging keeps comparison and suppresses warnings"], "generated": [{"profile": "shadow", "witness": "invalid-logging-compares-without-warning"}], "vectors": [], "contracts": ["C60"], "models": [ - {"ref": "formal/models/dialcache-shadow-conformance.qnt:invalidLoggingCannotBeCaptured", "scope": "An invalid logging flag is never captured as enabled. Admission still records one configuration error, comparison reaches mismatch, and no warning is emitted."}, - {"ref": "formal/models/dialcache-shadow-conformance.qnt:invalidLoggingKeepsComparisonWithoutWarningTest", "scope": "An invalid logging flag is never captured as enabled. Admission still records one configuration error, comparison reaches mismatch, and no warning is emitted."}]}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:capturedLoggingFollowsLayers", "scope": "An invalid logging input is never captured as an enabled leaf: each captured leaf is on only for a valid input in force. Admission still records one configuration error, comparison reaches mismatch, and no warning is emitted."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:invalidLoggingKeepsComparisonWithoutWarningTest", "scope": "An invalid logging input is never captured as an enabled leaf. Admission still records one configuration error, comparison reaches mismatch, and no warning is emitted."}]}, {"id": "W01.key-identity", "rule": "Construct interoperable logical, value, and watermark keys", "scenarios": [], "generated": [], "vectors": ["protocol/keyVectors/*"], "contracts": ["W01"], "generatedVectors": [{"artifact": "formal/generated/quint-key-vectors.json", "group": "keyVectors", "name": "*"}], "models": [ {"ref": "formal/models/dialcache-key-protocol.qnt:successfulKeysHaveOneOperationDelimiter", "scope": "UTF-8 percent escaping with uppercase hex and URI-safe punctuation, one operation delimiter, ordered raw arguments, frame suffix, and tracked watermark grouping; all ASCII plus bounded Unicode sequences moved through every identity/argument component. Caller-supplied configuration omission belongs to native construction tests."}, {"ref": "formal/models/dialcache-key-protocol.qnt:encodingSeparatesReservedDelimitersTest", "scope": "UTF-8 percent escaping with uppercase hex and URI-safe punctuation, one operation delimiter, ordered raw arguments, frame suffix, and tracked watermark grouping; all ASCII plus bounded Unicode sequences moved through every identity/argument component. Caller-supplied configuration omission belongs to native construction tests."}, @@ -1063,13 +1063,60 @@ {"ref": "formal/models/dialcache-recovery-read-conformance.qnt:freshCompressedReadReportsDecompressionTest", "scope": "Previously compressed inputs emit exact decompressed or fallback_raw diagnostics at decode start; later public completion distinguishes successful hit from source fallback while new compression writes remain disabled."}, {"ref": "formal/models/dialcache-recovery-read-conformance.qnt:corruptCompressedReadReportsRawFallbackTest", "scope": "Previously compressed inputs emit exact decompressed or fallback_raw diagnostics at decode start; later public completion distinguishes successful hit from source fallback while new compression writes remain disabled."}], "definitions": [ {"ref": "formal/models/kernel/payloads.qnt:decodeOutcome", "scope": "Previously compressed inputs emit exact decompressed or fallback_raw diagnostics at decode start; later public completion distinguishes successful hit from source fallback while new compression writes remain disabled."}]}, - {"id": "C60.logging-explicit-off", "rule": "Explicitly disabling mismatch logging suppresses warnings while preserving a confirmed mismatch verdict.", "contracts": ["C60"], "scenarios": ["mismatch warning omitted with mismatch verdict"], "generated": [{"profile": "shadow", "witness": "mismatch-logging:false"}, {"profile": "shadow", "witness": "explicit-false-shadow-logging-overrides-enabled-default"}], "vectors": [], "evidenceNotes": ["A confirmed mismatch with an explicitly disabled logging flag has no warning. Omitted/default logging and malformed flags have separate named cases."], "quintReplays": ["shadow/explicitFalseLoggingOverridesEnabledDefaultTest"], "models": [ - {"ref": "formal/models/dialcache-shadow-conformance.qnt:warningsRequireMismatch", "scope": "A confirmed mismatch with an explicitly disabled logging flag has no warning. Omitted/default logging and malformed flags have separate named cases."}, + {"id": "C60.logging-explicit-off", "rule": "Explicitly disabling mismatch logging suppresses warnings while preserving a confirmed mismatch verdict.", "contracts": ["C60"], "scenarios": ["mismatch warning omitted with mismatch verdict"], "generated": [{"profile": "shadow", "witness": "mismatch-logging:false"}, {"profile": "shadow", "witness": "explicit-false-shadow-logging-overrides-enabled-default"}], "vectors": [], "evidenceNotes": ["A confirmed mismatch under an explicit false logMismatches has no warning. Omitted/default logging and malformed inputs have separate named cases."], "quintReplays": ["shadow/explicitFalseLoggingOverridesEnabledDefaultTest"], "models": [ + {"ref": "formal/models/dialcache-shadow-conformance.qnt:warningsRequireMismatch", "scope": "A confirmed mismatch under an explicit false logMismatches has no warning. Omitted/default logging and malformed inputs have separate named cases."}, {"ref": "formal/models/dialcache-shadow-conformance.qnt:explicitFalseLoggingOverridesEnabledDefaultTest", "scope": "A logging-enabled fixture receives an explicit false runtime reply before admission. Confirmed mismatch and age metric remain, while warning and configuration-error counts remain zero."}], "definitions": [ {"ref": "formal/models/dialcache-shadow-conformance.qnt:initialize", "scope": "A logging-enabled fixture receives an explicit false runtime reply before admission. Confirmed mismatch and age metric remain, while warning and configuration-error counts remain zero."}, {"ref": "formal/models/dialcache-shadow-conformance.qnt:setLogging", "scope": "A logging-enabled fixture receives an explicit false runtime reply before admission. Confirmed mismatch and age metric remain, while warning and configuration-error counts remain zero."}, {"ref": "formal/models/kernel/shadow.qnt:beginDark", "scope": "A logging-enabled fixture receives an explicit false runtime reply before admission. Confirmed mismatch and age metric remain, while warning and configuration-error counts remain zero."}, {"ref": "formal/models/kernel/shadow.qnt:recordVerdict", "scope": "A logging-enabled fixture receives an explicit false runtime reply before admission. Confirmed mismatch and age metric remain, while warning and configuration-error counts remain zero."}]}, + {"id": "C60.warning-field-groups", "rule": "A confirmed-mismatch warning carries exactly the field groups its captured leaves name: the value fields with value, the paths and count at diff paths, the entries too at values, each from one rendering of each side, with the verdict's age.", "contracts": ["C60"], "scenarios": [], "generated": [{"profile": "shadow", "witness": "warning-fields:value"}, {"profile": "shadow", "witness": "warning-fields:paths"}, {"profile": "shadow", "witness": "warning-fields:values"}, {"profile": "shadow", "witness": "warning-fields:value+paths"}, {"profile": "shadow", "witness": "warning-fields:value+values"}], "vectors": [], "quintReplays": ["shadow/valueAloneLogsBothRenderingsTest", "shadow/valueWithDiffPathsLogsRenderingsAndPathsTest", "shadow/falseShorthandWithDiffPathsLogsPathsOnlyTest", "shadow/differingScalarsLogOneRootEntryTest"], "models": [ + {"ref": "formal/models/dialcache-shadow-conformance.qnt:warningFieldsMatchCapturedLeaves", "scope": "Each warning's value fields, diff level and diff fields restated from its verdict receipt: the leaves the job captured, the diff hook, the C0's decoded value and the source's result. Scalar values only (D12)."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:warningAgeIsVerdictAge", "scope": "A warning's cachedValueAgeSeconds is the age the same verdict sampled into the shadowAge channel, clamped at zero."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:valueAloneLogsBothRenderingsTest", "scope": "value alone: both value fields, no diff field."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:valueWithDiffPathsLogsRenderingsAndPathsTest", "scope": "value with diff paths: both value fields, the paths and count, no diffJson."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:falseShorthandWithDiffPathsLogsPathsOnlyTest", "scope": "diff paths alone: the paths and count only."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:differingScalarsLogOneRootEntryTest", "scope": "logMismatches true: every field, the one root entry carrying both sides."}], "definitions": [ + {"ref": "formal/models/kernel/shadow.qnt:warningOf", "scope": "The warning content over two scalar values, judged at the verdict from the captured leaves."}, + {"ref": "formal/models/kernel/shadow.qnt:recordVerdict", "scope": "A confirmed mismatch warns once when either captured leaf is on, beside the age it samples."}]}, + {"id": "C60.shorthand-table", "rule": "logMismatches expands in its own layer before the leaf-wise merge: a leaf written beside it overrides its expansion, a runtime shorthand replaces both inherited leaves, and a runtime leaf written off replaces only its own inherited leaf.", "contracts": ["C60"], "scenarios": [], "generated": [{"profile": "shadow", "witness": "shorthand-row:1"}, {"profile": "shadow", "witness": "shorthand-row:2"}, {"profile": "shadow", "witness": "shorthand-row:3"}, {"profile": "shadow", "witness": "shorthand-row:4"}, {"profile": "shadow", "witness": "shorthand-true-with-value-false-logs-diff-only"}, {"profile": "shadow", "witness": "shorthand-false-with-diff-paths-logs-paths-only"}, {"profile": "shadow", "witness": "runtime-false-shorthand-overrides-default-diff"}, {"profile": "shadow", "witness": "runtime-true-shorthand-overrides-default-value"}, {"profile": "shadow", "witness": "written-off-leaf:diff"}, {"profile": "shadow", "witness": "written-off-leaf:value"}, {"profile": "shadow", "witness": "runtime-off-diff-overrides-default-diff"}, {"profile": "shadow", "witness": "runtime-false-value-overrides-default-value"}], "vectors": [], "quintReplays": ["shadow/trueShorthandWithValueFalseLogsDiffOnlyTest", "shadow/falseShorthandWithDiffPathsLogsPathsOnlyTest", "shadow/runtimeFalseShorthandOverridesDefaultDiffTest", "shadow/runtimeTrueShorthandOverridesDefaultValueTest", "shadow/runtimeOffDiffOverridesDefaultDiffTest", "shadow/runtimeFalseValueOverridesDefaultValueTest"], "models": [ + {"ref": "formal/models/dialcache-shadow-conformance.qnt:capturedLoggingFollowsLayers", "scope": "At admission, each captured leaf follows its input in force, written independently of the kernel: the overlay's own leaf, else its shorthand's expansion, else the default's leaf, else its shorthand's."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:trueShorthandWithValueFalseLogsDiffOnlyTest", "scope": "Row 1: logMismatches true with value false logs diff values without the value fields."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:falseShorthandWithDiffPathsLogsPathsOnlyTest", "scope": "Row 2: logMismatches false with diff paths logs the paths only."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:runtimeFalseShorthandOverridesDefaultDiffTest", "scope": "Row 3: a runtime false shorthand over a default diff paths leaf logs nothing."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:runtimeTrueShorthandOverridesDefaultValueTest", "scope": "Row 4: a runtime true shorthand over a default value false leaf logs everything."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:runtimeOffDiffOverridesDefaultDiffTest", "scope": "A runtime diff off leaf over a logMismatches true default logs the value fields without the diff fields."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:runtimeFalseValueOverridesDefaultValueTest", "scope": "A runtime value false leaf over a logMismatches true default logs the diff values without the value fields."}], "definitions": [ + {"ref": "formal/models/kernel/mismatch-logging.qnt:expandLogging", "scope": "One layer's shorthand stands for both leaves and a leaf written beside it overrides that."}, + {"ref": "formal/models/kernel/mismatch-logging.qnt:mergeLogging", "scope": "The runtime layer wins leaf by leaf over the defaults, each layer expanded first."}]}, + {"id": "C60.invalid-input-isolation", "rule": "An invalid mismatch-logging input acts off and owes one configuration error, while a valid leaf beside it, or written beside a malformed shorthand, stays on.", "contracts": ["C60"], "scenarios": [], "generated": [{"profile": "shadow", "witness": "invalid-leaf-keeps-valid-leaf"}, {"profile": "shadow", "witness": "invalid-shorthand-keeps-written-leaf"}], "vectors": [], "quintReplays": ["shadow/invalidDiffLeafKeepsValueLeafTest", "shadow/invalidValueLeafKeepsDiffLeafTest", "shadow/invalidShorthandKeepsDiffLeafBesideItTest"], "models": [ + {"ref": "formal/models/dialcache-shadow-conformance.qnt:capturedLoggingFollowsLayers", "scope": "Per leaf: a leaf whose input in force is not a valid value is captured off at admission, whatever the other leaf captures."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:invalidDiffLeafKeepsValueLeafTest", "scope": "An invalid diff leaf beside value true: one configuration error, a value-only warning."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:invalidValueLeafKeepsDiffLeafTest", "scope": "An invalid runtime value leaf over a default logMismatches true, beside diff paths: one configuration error, a paths-only warning."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:invalidShorthandKeepsDiffLeafBesideItTest", "scope": "A malformed logMismatches beside diff values: one configuration error, a diff-values warning without the value fields."}], "definitions": [ + {"ref": "formal/models/kernel/mismatch-logging.qnt:resolveLogging", "scope": "Each leaf resolves on its own; one configuration error covers every invalid input."}, + {"ref": "formal/models/kernel/shadow.qnt:beginDark", "scope": "The configuration error is reported once where the hook and the ramp admit the job."}]}, + {"id": "C60.logging-outcomes", "rule": "Each warning attempt reports one shadowMismatchLogging outcome and logging off reports none: hook_error for the throwing diff hook, render_error for a side with no JSON rendering, logger_error for a failed logger, else logged; a failure nulls the fields it feeds.", "contracts": ["C60"], "scenarios": [], "generated": [{"profile": "shadow", "witness": "logging-outcome:logged"}, {"profile": "shadow", "witness": "logging-outcome:hook_error"}, {"profile": "shadow", "witness": "logging-outcome:render_error"}, {"profile": "shadow", "witness": "logging-outcome:logger_error"}, {"profile": "shadow", "witness": "unrenderable-source-logs-null"}, {"profile": "shadow", "witness": "logger-error-keeps-warning-content"}, {"profile": "shadow", "witness": "hook-error-nulls-diff-fields"}, {"profile": "shadow", "witness": "render-error-nulls-unrenderable-source"}, {"profile": "shadow", "witness": "logger-error-still-delivers-warning"}], "vectors": [], "evidenceNotes": ["The model's diff hook only throws; a hook's successful entries and an unrenderable hook value stay with the native tests."], "quintReplays": ["shadow/hookErrorNullsDiffFieldsTest", "shadow/renderErrorNullsUnrenderableSourceTest", "shadow/loggerErrorStillDeliversWarningTest"], "models": [ + {"ref": "formal/models/dialcache-shadow-conformance.qnt:oneLoggingOutcomePerWarning", "scope": "As many logging outcomes as warnings, and the warning counter agrees."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:noWarningWithLoggingOff", "scope": "The warnings are exactly the confirmed mismatches whose job captured either leaf on, in verdict order."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:hookErrorNullsDiffFieldsTest", "scope": "The throwing hook nulls every diff field and never falls back to the walk; the value fields still render."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:renderErrorNullsUnrenderableSourceTest", "scope": "A source result with no JSON rendering nulls its value field and every diff field (D13)."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:loggerErrorStillDeliversWarningTest", "scope": "A logger that throws received the full warning; only the outcome says logger_error."}], "definitions": [ + {"ref": "formal/models/kernel/shadow.qnt:loggingOutcomeOf", "scope": "The construction failure the warning names, else logger_error, else logged."}, + {"ref": "formal/models/kernel/shadow.qnt:recordMismatchWarning", "scope": "One warning and one outcome are recorded together."}]}, + {"id": "C60.outcome-precedence", "rule": "hook_error outranks render_error, which outranks logger_error; the diff hook runs only when diff is above off.", "contracts": ["C60"], "scenarios": [], "generated": [{"profile": "shadow", "witness": "hook-error-over-render-error"}, {"profile": "shadow", "witness": "hook-error-over-logger-error"}, {"profile": "shadow", "witness": "render-error-over-logger-error"}, {"profile": "shadow", "witness": "hook-skipped-at-diff-off"}, {"profile": "shadow", "witness": "hook-error-outranks-both-failures"}], "vectors": [], "quintReplays": ["shadow/hookErrorOutranksRenderErrorTest", "shadow/hookErrorOutranksLoggerErrorTest", "shadow/renderErrorOutranksLoggerErrorTest", "shadow/hookErrorOutranksBothFailuresTest", "shadow/hookSkippedWithDiffOffTest"], "models": [ + {"ref": "formal/models/dialcache-shadow-conformance.qnt:loggingOutcomePrecedence", "scope": "Each logging outcome and the warning's loggingOutcome restated from the verdict receipt's failure inputs in precedence order."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:hookErrorOutranksRenderErrorTest", "scope": "Hook and unrenderable source: hook_error."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:hookErrorOutranksLoggerErrorTest", "scope": "Hook and failed logger: hook_error."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:renderErrorOutranksLoggerErrorTest", "scope": "Unrenderable source and failed logger: render_error."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:hookErrorOutranksBothFailuresTest", "scope": "All three failures: hook_error."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:hookSkippedWithDiffOffTest", "scope": "With value alone the hook never runs, so the unrenderable source is the render_error the warning names."}], "definitions": [ + {"ref": "formal/models/kernel/shadow.qnt:warningOf", "scope": "hook_error is judged before render_error, and the hook only with diff on."}]}, + {"id": "W10.mismatch-paths", "rule": "The built-in mismatch walk over two JSON values: RFC 6901 pointers, members in UTF-16 order, the array rule, the root and absent-side cases, numbers compared as doubles and never equal to a bool, the depth cap and the entry, pointer and side caps with the uncut count.", "contracts": ["W10", "C60"], "scenarios": [], "generated": [{"profile": "shadow", "witness": "identical-renderings-log-no-paths"}, {"profile": "shadow", "witness": "differing-renderings-log-root-entry"}], "vectors": ["protocol/mismatchPathVectors/*"], "quintReplays": ["shadow/identicalRenderingsLogNoPathsTest", "shadow/differingScalarsLogOneRootEntryTest"], "models": [ + {"ref": "formal/models/dialcache-shadow-conformance.qnt:warningFieldsMatchCapturedLeaves", "scope": "Quint models only the scalar cases the shadow profile reaches (D12): no entry for identical renderings, else one root entry carrying both sides. The walk over nested values is the vectors' (D8)."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:identicalRenderingsLogNoPathsTest", "scope": "A comparator that calls equal values unequal: empty paths, count 0."}, + {"ref": "formal/models/dialcache-shadow-conformance.qnt:differingScalarsLogOneRootEntryTest", "scope": "Two different scalars: one entry at the root with both sides."}], "definitions": [ + {"ref": "formal/models/kernel/shadow.qnt:warningOf", "scope": "The walk's scalar cases over the model's values."}]}, {"id": "C40.fenced-candidate", "rule": "A fenced initial value is not retained or decoded for recovery; an allowed failure remains the original source error.", "contracts": ["C40"], "scenarios": [], "generated": [{"profile": "recovery", "witness": "fenced-read-does-not-retain"}], "vectors": [], "provenance": [{"path": "formal/guides/SPEC.md", "section": "Recovery, shadow work, and invalidation races"}, {"path": "typescript/src/internal/redis-cache.ts", "lines": [169, 198, 210, 224, 228], "scope": "Fresh/recovery acquisition and pre/post-decode age checks."}], "quintReplays": ["recovery/fencedInitialBytesNeverRecoverTest"], "models": [ {"ref": "formal/models/dialcache-recovery-conformance.qnt:fencedInitialBytesNeverRecoverTest", "scope": "A fenced initial value is not retained or decoded for recovery; an allowed failure remains the original source error."}, {"ref": "formal/models/dialcache-stale-recovery.qnt:fencedFrameNeverBecomesRecoveryCandidateTest", "scope": "A fenced initial value is not retained or decoded for recovery; an allowed failure remains the original source error."}]}, diff --git a/formal/catalogs/source-audit.json b/formal/catalogs/source-audit.json index 089574b9..69623d06 100644 --- a/formal/catalogs/source-audit.json +++ b/formal/catalogs/source-audit.json @@ -15,7 +15,7 @@ }, { "path": "docs/api.md", - "sha256": "04683617993432aea655d42d7fac51c2e9b3553d4f8c5535f1cc1a5e5e0ce56e", + "sha256": "315112302be59d3a112212ff05ba5f8d5c5e4d02ef220708c477f3a717082acb", "entries": [ {"title":"API reference","contracts":["B01"]}, {"title":"Imports","contracts":["B01"]}, @@ -90,7 +90,7 @@ }, { "path": "docs/configuration.md", - "sha256": "c11e862732d05f64314b07342a5499183a40fce7a87f7438d203e0c0a2dd90f4", + "sha256": "cc25114e5a5512d2dd48cf8b6c7466a395a0b449cf818946fd4d8b0c5acdd64e", "entries": [ {"title":"Configuration and rollout","contracts":["C16"]}, {"title":"What belongs where","contracts":["C16","B01"]}, @@ -165,7 +165,7 @@ }, { "path": "docs/languages/go.md", - "sha256": "d011de5bf43a253a0d3d2cd5d2cfdaa43926322ea65eba1781c5e8809a3c10c6", + "sha256": "514e3c9ab39e25d572b98dba915190ec2c368069f3d88a557657ffaeaffbf3c3", "entries": [ {"title":"Go integration","contracts":["B01"]}, {"title":"Installation and runtime","contracts":["B01","B02"]}, @@ -178,7 +178,7 @@ }, { "path": "docs/languages/python.md", - "sha256": "3cc5db10e5e69b517b32120c8f8be2c461c08a633b33d042922a1cee05cd6b32", + "sha256": "d6adc478599e184e585e15af8a3553f23e1749d041fe46f0358dbba1ee824b2e", "entries": [ {"title":"Python integration","contracts":["B01"]}, {"title":"Installation and runtime","contracts":["B01","B02"]}, @@ -192,7 +192,7 @@ }, { "path": "docs/languages/rust.md", - "sha256": "bd24a8859958f816750c1c4ce70e05f4f366f56d50ea8b491586e9c6441f4215", + "sha256": "ab4fd96e1664d9525adba0d80d9be6f209329d6ab1cf35091891f5c3c59a719f", "entries": [ {"title":"Rust integration","contracts":["B01"]}, {"title":"Installation and runtime","contracts":["B01","B02"]}, @@ -205,7 +205,7 @@ }, { "path": "docs/languages/typescript.md", - "sha256": "8f6706c33d8e712a9ef70acec8d212a4db20cd206c1b2d04a0c7b2f18f0f1bb7", + "sha256": "27a4546a5bef235b84ff89c27ab50b4fa62a84d4bebcc17b687aeb4b514e95ec", "entries": [ {"title":"TypeScript integration","contracts":["B01"]}, {"title":"Installation and runtime","contracts":["B01","B02"]}, @@ -217,7 +217,7 @@ }, { "path": "docs/maintainers.md", - "sha256": "b52155b4505aca801509f0a624ce20dfcf9fd7e3f9c687ed72ab748e7bcf6420", + "sha256": "61eda3022f04eb70fd70ff8d4ea49959914599dc02802c1778e56e9b0c09adcf", "entries": [ {"title":"Maintainer guide","contracts":["B01"]}, {"title":"Validation","contracts":["B01"]}, @@ -236,7 +236,7 @@ }, { "path": "docs/observability.md", - "sha256": "bdafd62305526062fe913fada398d9356c4d081b6c2f1c2744b1cc7454bbe97c", + "sha256": "ed88c26971eae18b957c0852c02d5ea35fdb814458827d79c7807a7c42c0db19", "entries": [ {"title":"Observability","contracts":["C30","X01"]}, {"title":"Reading the signals","contracts":["C58"]}, @@ -283,7 +283,7 @@ }, { "path": "docs/shadow-validation.md", - "sha256": "98c932e4c75653c132d8a2408c3c8d64e4f498f88d01b56edae59a9e8c10d473", + "sha256": "e318acf6936ad4c9126363eff2c803d00a20596f92a00c543e645cb6ca765c34", "entries": [ {"title":"Shadow validation","contracts":["C47"]}, {"title":"What a check establishes","contracts":["C50","C51"]}, @@ -299,7 +299,7 @@ {"title":"Capacity, deadlines, and detachment","contracts":["C54","E03"]}, {"title":"Consistency modes and race boundaries","contracts":["C36","C51"]}, {"title":"Command amplification","contracts":["C49","C51","C52"]}, - {"title":"Confirmed mismatch logging","contracts":["C60","B03","X01"]}, + {"title":"Confirmed mismatch logging","contracts":["C60","W10","B03","X01"]}, {"title":"Metrics and shutdown","contracts":["C57","C58","C59","E03"]} ] }, @@ -319,7 +319,7 @@ }, { "path": "docs/upgrading.md", - "sha256": "a02cb92599073440b0bc6f693b064236759dced02d578591934830fc4c9ff425", + "sha256": "f35d61a95ac2f9ad9dcc5d1f86c5b151b1260f7e7679f75a161777cbe39e8e84", "entries": [ {"title":"Upgrading","contracts":["E05"]}, {"title":"Tracked protocol cutover","contracts":["E05","W09"]}, @@ -327,6 +327,7 @@ {"title":"Custom Redis adapters","contracts":["E01","C55"]}, {"title":"Stale retention and downgrades","contracts":["E05","C37"]}, {"title":"Compression and value schemas","contracts":["E05","W06","W08"]}, + {"title":"Shadow mismatch warnings","contracts":["C60","X01"]}, {"title":"Metric migrations","contracts":["X01"]} ] }, @@ -390,7 +391,7 @@ }, { "path": "typescript/test/datadog.test.ts", - "sha256": "8b50a81e6baa8d3d1eb4854cb9d84d83cc48493bca4efc0b656e0f0714f66d65", + "sha256": "2c953d1fa3f861dd2d036105856a4d8bfed72887b8fc77f0c7354ababff7a747", "entries": [ {"title":"maps every DialCache metric to the default metric namespace with exact names, values, and tags","contracts":["X01"]}, {"title":"`uses ${observationMetricType} for every observation, so switching modes requires a new namespace`","contracts":["X01"]}, @@ -453,7 +454,7 @@ }, { "path": "typescript/test/dialcache-config-ramp.test.ts", - "sha256": "8375f7c9f7cb492b59177a263ac1b3e2279d85577fd9a16c461ec118bbea3dfa", + "sha256": "cd4b45c7d09cf3e1e5edcc9c4f33b586ee6e160a9bdb1d4692afd4b6124a2fed", "entries": [ {"title":"rejects the removed public ramp sampler override","contracts":["B01"]}, {"title":"preserves requestLocal omission until baseline and runtime config are merged","contracts":["C16"]}, @@ -461,7 +462,10 @@ {"title":"preserves stale-on-error omission and explicit disable values","contracts":["C16","C17"]}, {"title":"preserves shadow omission and explicit kill-switch values","contracts":["C16","C17"]}, {"title":"clones the supplied shadow policy","contracts":["B01"]}, + {"title":"clones the supplied mismatchLogging leaves","contracts":["B01"]}, + {"title":"leaves mismatchLogging leaf values unvalidated until static capture or runtime resolution","contracts":["B01"]}, {"title":"captures an immutable default policy snapshot when the use case is registered","contracts":["C18","B01"]}, + {"title":"captures an immutable mismatchLogging snapshot when the use case is registered","contracts":["C18","B01"]}, {"title":"enables request-local caching without TTL or ramp policy","contracts":["C07"]}, {"title":"fetches runtime config once while traversing request-local, local, and remote layers","contracts":["C18"]}, {"title":"uses a deterministic ramp sample per cache key and layer","contracts":["W03"]}, @@ -632,7 +636,7 @@ }, { "path": "typescript/test/dialcache-metrics.test.ts", - "sha256": "ce0e770bbaf97621c97ea329fd513e27810a9fc27aa6f736f1ad1e1f8ffb528e", + "sha256": "866891f1f462f900f3dc204fd0777ab81e8d76e2b2da1e9f99a42d5dc5ef2e1c", "entries": [ {"title":"accepts only the bounded miss reasons, not inherited keys or non-string values","contracts":["C55","C58"]}, {"title":"consumes rejecting thenables returned by every metrics method without awaiting them","contracts":["C30","B02"]}, @@ -648,6 +652,7 @@ {"title":"reports request-local cache activity and request-scoped coalescing with bounded labels","contracts":["C58"]}, {"title":"fails open when an injected metrics adapter throws","contracts":["C30"]}, {"title":"fails open when the coalesced metrics hook throws","contracts":["C30"]}, + {"title":"fails open when the shadow mismatch logging metrics hook throws","contracts":["C30","C60"]}, {"title":"classifies disabled cache skips by reason","contracts":["C58"]}, {"title":"reports invalid stale-on-error policy without disabling fresh Redis","contracts":["C21","C58"]}, {"title":"labels cache errors separately from fallback errors","contracts":["C58"]}, @@ -663,10 +668,19 @@ }, { "path": "typescript/test/dialcache-observability-internals.test.ts", - "sha256": "70e8901dd77e950362f14085989cd0560b9c9bb01f013cdca4bf5b3a9903eb8e", + "sha256": "6c8965b7c155d8740d47424c70d4bf43543fcd0c6aac15b267d4e7cc9be02041", "entries": [ {"title":"merges every runtime policy leaf independently","contracts":["C16"]}, {"title":"preserves omitted requestLocal and coalesce through a runtime merge","contracts":["C16"]}, + {"title":"resolves %s to the recorded effective leaves","contracts":["C16","C60"]}, + {"title":"inherits an omitted mismatchLogging leaf from the default","contracts":["C16"]}, + {"title":"lets a runtime mismatchLogging leaf override its default leaf","contracts":["C16"]}, + {"title":"keeps a malformed runtime logMismatches for admission while its same-object leaves win","contracts":["C21","C60"]}, + {"title":"resolves %s to the effective mismatch-logging leaves","contracts":["C21","C60"]}, + {"title":"rejects the removed shadowRamp field in a provider result","contracts":["B01","X01"]}, + {"title":"resolves every merged combination of shorthand and leaves as each layer expanded first","contracts":["C16","C60"]}, + {"title":"expands the logMismatches shorthand before the same group's leaves override it","contracts":["C16","C60"]}, + {"title":"rejects a provider result whose mismatchLogging is %s","contracts":["C20"]}, {"title":"keeps invalid stale-on-error policy diagnostic-only in remote resolution","contracts":["C21"]} ] }, @@ -744,7 +758,7 @@ }, { "path": "typescript/test/dialcache-shadow-confirmation.test.ts", - "sha256": "98d38eaf23999cdb1072ba0f9136c7ef028f59f45f58984b41fe2969920b36b1", + "sha256": "db7d43c1428fbf4a84bcde1dc3876a1b56508b26309c25faf99d4ae95d7145a1", "entries": [ {"title":"skips confirmation when the served Redis payload semantically matches SoT","contracts":["C50"]}, {"title":"confirms a mismatch only when C1 is byte-identical to the served C0","contracts":["C51"]}, @@ -752,8 +766,25 @@ {"title":"does not log confirmed mismatches when logging is omitted","contracts":["C60"]}, {"title":"logs one bounded JSON warning for a served-hit confirmed mismatch","contracts":["C60"]}, {"title":"logs the logical key and compared values for a ramped-down confirmed mismatch","contracts":["C60"]}, - {"title":"falls back to a metadata warning when JSON detail construction throws","contracts":["C60","B03"]}, + {"title":"logs every requested field as null with render_error when warning construction throws","contracts":["C60","B03"]}, {"title":"clamps the logical key and both JSON strings before handing details to the logger","contracts":["X01"]}, + {"title":"logs only the paths and their count at diff paths","contracts":["C60"]}, + {"title":"logs the paths and diffJson without whole-value JSON at diff values with value false","contracts":["C60"]}, + {"title":"logs only the whole-value JSON and never runs the diff hook at value true alone","contracts":["C60"]}, + {"title":"logs the exact value age the verdict observed as cachedValueAgeSeconds","contracts":["C57","C60"]}, + {"title":"logs the documented GetUser example at diff $diff","contracts":["C60","X01"]}, + {"title":"emits one logged event per confirmed mismatch after its verdict and none for a match","contracts":["C60","X01"]}, + {"title":"rejects %s as shadowMismatchDiff where each API captures it","contracts":["B01"]}, + {"title":"logs shadowMismatchDiff entries in place of the built-in walk through %s()","contracts":["C60"]}, + {"title":"logs null diff fields and hook_error without touching the verdict when the hook $name","contracts":["C30","C60"]}, + {"title":"logs render_error with null source and diff fields for a bigint source value under $name","contracts":["C60","B03"]}, + {"title":"logs a cached undefined negative-cache entry as an absent side, not a render failure","contracts":["C60","B03"]}, + {"title":"passes an absent cached side to shadowMismatchDiff as the raw undefined","contracts":["C60","B03"]}, + {"title":"logs hook paths without rendering values at diff paths, even for a bigint source value","contracts":["C60","B03"]}, + {"title":"records $outcome and leaves the caller unaffected when the logger $name","contracts":["C30","C60"]}, + {"title":"records $outcome rather than logger_error when $name and the logger $loggerFailure","contracts":["C30","C60"]}, + {"title":"records $outcome at once when $name while the logger's promise is still pending","contracts":["C30","C60"]}, + {"title":"records no logging outcome while a logger's promise never settles","contracts":["C30","C60"]}, {"title":"reports superseded when C1 is $name","contracts":["C51"]}, {"title":"reports superseded when C1 becomes a typed watermark miss","contracts":["C51"]}, {"title":"retains a future-dated C1 for payload confirmation and attributes its offset to remote_shadow","contracts":["C51","C57"]}, @@ -788,6 +819,10 @@ {"title":"does not dark-read through invalid runtime policy or provider failure","contracts":["C20","C47"]}, {"title":"fails closed for invalid runtime mismatch logging without suppressing the metric","contracts":["C60"]}, {"title":"treats DialCacheKeyConfig.disabled() as a complete shadow kill switch","contracts":["C17","C47"]}, + {"title":"keeps the valid default leaf and records one config_resolution for an invalid runtime leaf","contracts":["C21","C60"]}, + {"title":"records one config_resolution for a malformed runtime shorthand and logs the leaves beside it","contracts":["C21","C60"]}, + {"title":"logs $name","contracts":["C16","C60"]}, + {"title":"lets DialCacheKeyConfig.disabled() override every inherited mismatch-logging leaf","contracts":["C16","C60"]}, {"title":"keeps partial Redis-serving and shadow-observation cohorts independent","contracts":["W03","C47"]}, {"title":"preserves remote-ramp-zero concurrency while deduplicating only shadow work","contracts":["C49"]}, {"title":"lets the caller's SoT result populate active local cache, never the dark Redis value","contracts":["C49"]}, @@ -879,7 +914,7 @@ }, { "path": "typescript/test/dialcache-stale-recovery-policy.test.ts", - "sha256": "0cd3257da5c88748802579c663b3c5e3bad13a53fadb8e1ed448a1b2a92a65e8", + "sha256": "aba1c3da6a6a542bdb53d20a3a4fb6c804b18d2370dfa7774f45160c005ab3b3", "entries": [ {"title":"validates the instance predicate at construction","contracts":["B01"]}, {"title":"validates use-case predicates when each API captures them","contracts":["B01"]}, @@ -942,7 +977,7 @@ }, { "path": "typescript/test/prometheus.test.ts", - "sha256": "1c081c00ca8642c8ed5e7a3fd5a5faa2d5d06d333f99c2fb0832ead0057c1602", + "sha256": "b459867a7646b3096985a9e42318af7eaf1d4a6a84c64b6e7145837558100296", "entries": [ {"title":"keeps default DialCache construction detached from Prometheus","contracts":["X01"]}, {"title":"registers collectors only in the caller-owned registry","contracts":["X01"]}, @@ -952,12 +987,14 @@ {"title":"exports every bounded disabled reason without rewriting labels","contracts":["X01"]}, {"title":"exports every bounded miss reason without rewriting labels","contracts":["X01"]}, {"title":"exports every bounded shadow-validation outcome without adding cache identity or layer labels","contracts":["X01"]}, + {"title":"exports every bounded shadow-mismatch-logging outcome without adding cache identity or layer labels","contracts":["X01"]}, {"title":"exports every bounded stale-recovery outcome without adding cache identity or layer labels","contracts":["X01"]}, {"title":"exports every bounded compression outcome without rewriting labels","contracts":["X01"]}, {"title":"uses the existing layer label for detached Redis telemetry","contracts":["X01"]}, {"title":"reuses existing collectors when multiple adapters share a registry","contracts":["X01"]}, {"title":"rejects a pre-registered legacy miss collector before registering anything","contracts":["X01"]}, {"title":"reuses a pre-registered current-schema miss collector","contracts":["X01"]}, + {"title":"reuses a pre-registered current-schema shadow-mismatch-logging collector","contracts":["X01"]}, {"title":"`rejects an existing collector with incompatible ${schemaPart} before registering anything`","contracts":["X01"]}, {"title":"rejects exemplar-enabled collectors before registering anything","contracts":["X01"]}, {"title":"treats an explicitly undefined exemplar mode as disabled","contracts":["B01"]}, @@ -1043,12 +1080,45 @@ }, { "path": "typescript/test/shadow-log-json.test.ts", - "sha256": "97d2a80d23bbe76574f97b0f569307a203279de4fa770a253603648325801467", + "sha256": "2dab2b5558563125f731410b170a41c8c216ff28d54a012b3744ce387c99213f", "entries": [ {"title":"uses native JSON for the values supplied to the comparator","contracts":["B03","X01"]}, {"title":"returns null independently for values that native JSON cannot serialize","contracts":["B03","X01"]}, {"title":"catches user JSON hook failures","contracts":["C30","B03"]}, - {"title":"byte-clamps keys and JSON without splitting UTF-8 sequences","contracts":["X01"]} + {"title":"byte-clamps keys and JSON without splitting UTF-8 sequences","contracts":["X01"]}, + {"title":"computes the expected entries for $name","contracts":["W10","C60","X01"]}, + {"title":"renders the expected diff fields for $name","contracts":["W10","C60","X01"]}, + {"title":"walks own members whose names collide with Object.prototype","contracts":["W10","C60","B03"]}, + {"title":"compares only own members in a depth-capped pair","contracts":["W10","C60","B03"]}, + {"title":"cuts the entries at 64 in walk order and counts every difference","contracts":["W10","C60","X01"]}, + {"title":"clamps each pointer to 512 bytes without splitting UTF-8 sequences","contracts":["W10","X01"]}, + {"title":"builds each kept pointer only as far as its clamp reads, however long the member names above it","contracts":["W10","X01"]}, + {"title":"replaces each side over 1 KiB with its clamped JSON text","contracts":["W10","X01"]}, + {"title":"caps the walk at 32 reference tokens however deep the difference lies","contracts":["W10","C60","X01"]}, + {"title":"compares equal containers past the depth cap regardless of member order and depth","contracts":["W10","C60","X01"]}, + {"title":"renders exactly the fields requested by %s","contracts":["C60"]}, + {"title":"renders no entries for byte-identical renderings","contracts":["C60","B03"]}, + {"title":"renders no entries for the documented Date row","contracts":["C60","B03"]}, + {"title":"reports the source whole at the root for the documented undefined row","contracts":["W10","C60","B03"]}, + {"title":"renders no entries for a Date against its ISO string","contracts":["C60","B03"]}, + {"title":"nulls the fields fed by a root %s with render_error","contracts":["B03","X01"]}, + {"title":"reports an absent root as one side at the root pointer, not as a render failure","contracts":["W10","C60","B03"]}, + {"title":"runs each side's toJSON once for every field it feeds","contracts":["C60","B03"]}, + {"title":"clamps diffJson to 8 KiB with the truncation marker","contracts":["X01"]}, + {"title":"fails every field %s requests closed to null, in rendering order","contracts":["C60"]}, + {"title":"clamps each built-in diffJson side to 1 KiB","contracts":["X01"]}, + {"title":"takes the entries from the hook and never runs the built-in walk","contracts":["C60"]}, + {"title":"renders neither root side when the hook supplies the entries and value is off","contracts":["C60","B03"]}, + {"title":"fails closed with hook_error and no fallback when the hook %s","contracts":["C60","B02"]}, + {"title":"logs the hook's entries as it returns them and reads only the kept ones","contracts":["C60"]}, + {"title":"consumes a rejecting hook result without an unhandled rejection","contracts":["C30","B02"]}, + {"title":"keeps the hook paths and nulls diffJson when a hook side is %s","contracts":["C60","B03"]}, + {"title":"stops rendering hook sides at the first side with no rendering","contracts":["C60","B03"]}, + {"title":"omits absent hook sides from diffJson and renders present ones as JSON","contracts":["C60"]}, + {"title":"clamps hook paths and sides and cuts at 64 with the total count","contracts":["X01"]}, + {"title":"does not call the hook when diff is off","contracts":["C60"]}, + {"title":"keeps the hook entries when a root side has no rendering","contracts":["C60","B03"]}, + {"title":"reports hook_error ahead of render_error","contracts":["C60","X01"]} ] }, { @@ -1084,7 +1154,7 @@ "reviewedGuides": [ { "path": "formal/README.md", - "sha256": "8c3930f2938792bd470f8f7c862a5ecfe6c0cb2c5a43c587ca0a515d96f7424b", + "sha256": "a27fae8c54e7f257437d72738ef610cc309233066a9d0fccea26255630130c13", "entries": [ {"title":"Executable DialCache specification"}, {"title":"Start with your task"}, @@ -1108,7 +1178,7 @@ }, { "path": "formal/guides/AUTHORING.md", - "sha256": "378e920719b033ff0e713d8a0932c216922684aad14a2c0f06af23db1c6c093d", + "sha256": "51ba5d1f4a353059a085ea6c98243b0ca2050bfcf66ac8ed167d2527211846bb", "entries": [ {"title":"Reading and extending the DialCache models"}, {"title":"Reading a model"}, @@ -1130,7 +1200,7 @@ }, { "path": "formal/guides/BEHAVIOR.md", - "sha256": "5d6cb8d7e779cb99e6a60f6ea553b524d49c7a42b79a74f912a90ff44c5bafe8", + "sha256": "0a9404acd450a011913db08bcb6170f7023e069f3e8ee5ed127f116e96274b43", "entries": [ {"title":"Portable behavioral scenarios and generated profiles"}, {"title":"Scenario format"}, @@ -1257,7 +1327,7 @@ }, { "path": "formal/guides/CONTRACTS.md", - "sha256": "6b6c3a08412f1ad1da0700dd4777a7e61306b6b7559e3409b83b15dda75ff8c6", + "sha256": "b9bad07a7d1cc8485cf91dda40937231efb2548519c63ecdb4dee0e80b8627c1", "entries": [ {"title":"Portable contract inventory"}, {"title":"Where the boundary lies"}, @@ -1345,6 +1415,7 @@ "W07", "W08", "W09", + "W10", "E01", "E02", "E03", @@ -1360,7 +1431,7 @@ }, { "path": "formal/guides/FEATURE-COVERAGE.md", - "sha256": "8368ec455fa600c89821853926ca1c5025ba18bb6dacdc6d6a34794d9ade8afb", + "sha256": "a4bb78f80bff5982db8622802884c80e3ec3cec845fa10559c59591c2231fb19", "entries": [ {"title":"DialCache features and known behavioral corners"}, {"title":"What is accounted for"}, @@ -1436,7 +1507,7 @@ }, { "path": "formal/guides/GO-PARITY.md", - "sha256": "956b12eabff3716bac5f4c57d14315c792dc89524c224a261fd15508992ff63f", + "sha256": "0ab9505cb3cb1185887f080882bb50b16cf88dc7134305e144478ece178c7a7b", "entries": [ {"title":"Go parity acceptance"}, {"title":"Required evidence"}, @@ -1482,7 +1553,7 @@ }, { "path": "formal/guides/PORTING.md", - "sha256": "342cfd680eeba31a57c7cc790693fb21d56e358e20ce105a6a6ce28a9e511a7e", + "sha256": "fe6b5b4ca77793a8e660fe328571db1c0dacdebc78f0262dccacc5bb1af02508", "entries": [ {"title":"Implementing another DialCache port"}, {"title":"What a port supplies"}, @@ -1575,7 +1646,7 @@ }, { "path": "formal/guides/SPEC.md", - "sha256": "e1efa3b41827ba46de5701d7c08b599dd3c6d89b302fa5c17b9a091dfe3ace14", + "sha256": "d20e81545e58e181dc0632d90a8dbf90704d13b2f6a72212d78adb31085a5932", "entries": [ {"title":"DialCache behavioral specification"}, {"title":"Definition ownership and executable connections"}, @@ -1696,7 +1767,7 @@ }, { "path": "formal/guides/VALIDATION.md", - "sha256": "afc44715faa7df1512aa8ab1d013521319e97e9ce75725aa20b292459504b2bc", + "sha256": "da0f7d2bffaa110f7945d92897220366384dfc93d96af7171d809ffdda2e5a0e", "entries": [ {"title":"Validation and evidence"}, {"title":"Prerequisites"}, @@ -1729,7 +1800,7 @@ }, { "path": "go/README.md", - "sha256": "fa465c2a7e53b4fe3ea2045708510c81ea5304e62533f99726d9bcdf1ba9c079", + "sha256": "0d77c6caf885d508febae846a4a06dc33cd63e0d2b18faccdfc4ac8781d271fb", "entries": [ {"title":"DialCache for Go"}, {"title":"Use"}, diff --git a/formal/catalogs/witness-baseline.json b/formal/catalogs/witness-baseline.json index 34892e8c..09326114 100644 --- a/formal/catalogs/witness-baseline.json +++ b/formal/catalogs/witness-baseline.json @@ -7,7 +7,7 @@ "profiles": { "admission": { "sampledHistories": 128, - "corpusSha256": "27d0a291fdb2f93e01a01a0dc202ec951ab70006e5bebc377c1c2bd0a83db0de", + "corpusSha256": "17d6a0236ab3579ecfb7eb4b91d1ff188d445be6d99c83369ef8dedba140345c", "labels": { "outcome:match": 57, "outcome:mismatch": 35, @@ -32,7 +32,7 @@ }, "dark-layers": { "sampledHistories": 256, - "corpusSha256": "e30409007315d928e5ff104c3c11868f7b9664a75a2e1ca7cb90389ce13fa68d", + "corpusSha256": "a97efb58bb31066ad7b0369162bebd2e115da329cb686f9008b07035ee70114f", "labels": { "inspection-counts-process-followers-and-oldest": 0, "inspection-isolates-instances-and-wall-shifts": 0, @@ -464,90 +464,128 @@ }, "shadow": { "sampledHistories": 1024, - "corpusSha256": "d2027e4824ab64f7c15c9d9aceee09d3924476f932ed2aeef440719ef831eca5", + "corpusSha256": "371e6efc51bc4c10e29917916800b366ba0225d4fe3e3dc195144fbda2e76474", "labels": { - "outcome:match": 53, - "outcome:mismatch": 54, - "outcome:comparison_error": 40, - "outcome:superseded": 25, - "outcome:confirmation_error": 9, - "outcome:redis_error": 441, - "outcome:source_error": 636, - "outcome:timeout": 466, - "outcome:deserialization_error": 76, + "outcome:match": 59, + "outcome:mismatch": 60, + "outcome:comparison_error": 34, + "outcome:superseded": 19, + "outcome:confirmation_error": 11, + "outcome:redis_error": 465, + "outcome:source_error": 718, + "outcome:timeout": 486, + "outcome:deserialization_error": 89, "outcome:filled": 199, - "outcome:fill_error": 139, - "outcome:fill_fenced": 307, - "c0-before-source": 567, - "source-before-c0": 697, - "write-completes-after-timeout": 12, - "fixture:0": 73, - "fixture:1": 68, - "fixture:2": 72, - "fixture:3": 76, - "fixture:4": 85, - "fixture:5": 79, - "fixture:6": 75, - "fixture:7": 79, - "fixture:8": 102, - "fixture:9": 82, - "fixture:10": 76, + "outcome:fill_error": 146, + "outcome:fill_fenced": 338, + "c0-before-source": 644, + "source-before-c0": 776, + "write-completes-after-timeout": 14, + "fixture:0": 65, + "fixture:1": 73, + "fixture:2": 66, + "fixture:3": 69, + "fixture:4": 67, + "fixture:5": 65, + "fixture:6": 55, + "fixture:7": 77, + "fixture:8": 65, + "fixture:9": 64, + "fixture:10": 58, "text-to-binary-confirmation": 8, - "binary-to-text-confirmation": 6, - "different-bytes-same-value-superseded": 11, - "binary-c0-compares-decoded-value": 6, - "comparison-crosses-deadline:9": 14, - "comparison-crosses-deadline:10": 16, - "missing-hook-skips-job": 102, - "shadow-policy-skips-job:1": 343, - "shadow-policy-skips-job:2": 358, - "admitted-job-keeps-shadow-policy": 49, - "custom-equal-overrides-values": 28, - "custom-unequal-confirms-equal-values": 8, - "captured-logging:true": 5, - "captured-logging:false": 7, - "mismatch-logging:true": 26, - "mismatch-logging:false": 32, - "omitted-logging-defaults-off": 14, - "age-clamped-after-rollback": 12, - "age-at-verdict": 42, - "fixture:11": 75, - "fixture:12": 82, - "invalid-logging-compares-without-warning": 8, - "source-work-before-deadline-dispatches-read": 74, - "source-work-exhausts-deferred-job": 82, - "expired-job-source-resolve-keeps-deadline": 82, - "expired-job-source-reject-keeps-deadline": 81, - "enabled-logging-match-has-no-warning": 33, - "enabled-logging-superseded-has-no-warning": 8, - "dark-read-error-still-allows-source-result": 158, - "dark-source-error-never-decodes-or-fills": 636, - "equal-comparison-skips-c1": 53, - "fenced-c1-supersedes-without-repair": 10, - "fenced-dark-c0-can-fill-after-cutoff": 4, - "fill-serialization-error-preserves-caller": 78, - "fill-write-error-preserves-caller": 63, - "future-c1-confirms-payload-without-repair": 2, - "future-dark-c0-fills-without-decoding": 46, - "late-c0-cannot-start-new-shadow-work": 297, - "late-c1-cannot-emit-second-verdict": 8, - "late-shadow-decode-cannot-start-c1": 30, - "late-shadow-dump-cannot-dispatch-write": 15, - "late-shadow-write-cannot-change-caller-or-verdict": 12, - "present-undecodable-c0-is-not-repaired": 76, - "different-c1-bytes-supersede": 9, - "same-c1-bytes-confirm-mismatch": 54, + "binary-to-text-confirmation": 9, + "different-bytes-same-value-superseded": 4, + "binary-c0-compares-decoded-value": 1, + "comparison-crosses-deadline:9": 12, + "comparison-crosses-deadline:10": 20, + "missing-hook-skips-job": 65, + "shadow-policy-skips-job:1": 195, + "shadow-policy-skips-job:2": 171, + "admitted-job-keeps-shadow-policy": 26, + "custom-equal-overrides-values": 36, + "custom-unequal-confirms-equal-values": 4, + "captured-logging:true": 9, + "captured-logging:false": 15, + "mismatch-logging:true": 36, + "mismatch-logging:false": 25, + "omitted-logging-defaults-off": 7, + "age-clamped-after-rollback": 13, + "age-at-verdict": 47, + "fixture:11": 62, + "fixture:12": 50, + "invalid-logging-compares-without-warning": 3, + "source-work-before-deadline-dispatches-read": 62, + "source-work-exhausts-deferred-job": 50, + "expired-job-source-resolve-keeps-deadline": 50, + "expired-job-source-reject-keeps-deadline": 50, + "enabled-logging-match-has-no-warning": 45, + "enabled-logging-superseded-has-no-warning": 11, + "dark-read-error-still-allows-source-result": 117, + "dark-source-error-never-decodes-or-fills": 718, + "equal-comparison-skips-c1": 59, + "fenced-c1-supersedes-without-repair": 4, + "fenced-dark-c0-can-fill-after-cutoff": 9, + "fill-serialization-error-preserves-caller": 87, + "fill-write-error-preserves-caller": 64, + "future-c1-confirms-payload-without-repair": 6, + "future-dark-c0-fills-without-decoding": 41, + "late-c0-cannot-start-new-shadow-work": 318, + "late-c1-cannot-emit-second-verdict": 6, + "late-shadow-decode-cannot-start-c1": 21, + "late-shadow-dump-cannot-dispatch-write": 20, + "late-shadow-write-cannot-change-caller-or-verdict": 14, + "present-undecodable-c0-is-not-repaired": 89, + "different-c1-bytes-supersede": 13, + "same-c1-bytes-confirm-mismatch": 60, "omitted-shadow-logging-keeps-mismatch-metrics-only": 0, "explicit-false-shadow-logging-overrides-enabled-default": 0, "shadow-confirmation-past-freshness-preserves-payload-and-age": 0, "shadow-confirmation-rollback-keeps-payload-and-reports-offset": 0, "future-dark-c0-reports-offset-and-fills-semantic-miss": 0, - "late-dark-read-reports-offset-without-reviving-job": 0 + "late-dark-read-reports-offset-without-reviving-job": 0, + "fixture:13": 66, + "fixture:14": 60, + "fixture:15": 62, + "warning-fields:value": 12, + "warning-fields:paths": 10, + "warning-fields:values": 4, + "warning-fields:value+paths": 2, + "warning-fields:value+values": 9, + "logging-outcome:logged": 15, + "logging-outcome:hook_error": 3, + "logging-outcome:render_error": 11, + "logging-outcome:logger_error": 10, + "identical-renderings-log-no-paths": 1, + "differing-renderings-log-root-entry": 14, + "unrenderable-source-logs-null": 6, + "hook-error-over-render-error": 0, + "hook-error-over-logger-error": 3, + "render-error-over-logger-error": 7, + "hook-skipped-at-diff-off": 3, + "logger-error-keeps-warning-content": 10, + "shorthand-row:1": 0, + "shorthand-row:2": 3, + "shorthand-row:3": 2, + "shorthand-row:4": 1, + "written-off-leaf:diff": 4, + "written-off-leaf:value": 0, + "invalid-leaf-keeps-valid-leaf": 7, + "invalid-shorthand-keeps-written-leaf": 3, + "shorthand-true-with-value-false-logs-diff-only": 0, + "shorthand-false-with-diff-paths-logs-paths-only": 0, + "runtime-false-shorthand-overrides-default-diff": 0, + "runtime-true-shorthand-overrides-default-value": 0, + "runtime-off-diff-overrides-default-diff": 0, + "runtime-false-value-overrides-default-value": 0, + "hook-error-nulls-diff-fields": 0, + "render-error-nulls-unrenderable-source": 0, + "logger-error-still-delivers-warning": 0, + "hook-error-outranks-both-failures": 0 } }, "shadow-layers": { "sampledHistories": 256, - "corpusSha256": "6afffa10eb14b0fe44b31ed244d4c16a8ae448fc1999e8f9e20759043d8e814d", + "corpusSha256": "4bcd2342725d7e9b572ddb3b6c2346dafc0877907004859dd049ab7b2a00b84f", "labels": { "dark-present-c0-never-publishes-local": 0, "dark-local-source-publication-stops-later-shadow": 0, @@ -575,7 +613,7 @@ }, "shadow-read-deadlines": { "sampledHistories": 128, - "corpusSha256": "ea77c01153937b49307768d155de915cc8bc6fb7194ecc3df7fef34f0040a624", + "corpusSha256": "5cfd73ac4275e4d802d931b90da05d22655fd0ead77b084d8f891b71b019fc75", "labels": { "c0-read-timeout-retains-raw-capacity": 0, "c1-read-timeout-retains-raw-capacity": 0, diff --git a/formal/generated/CHALLENGES.md b/formal/generated/CHALLENGES.md index 86e17634..00bf8649 100644 --- a/formal/generated/CHALLENGES.md +++ b/formal/generated/CHALLENGES.md @@ -38,6 +38,17 @@ explains what a challenge measures. | `stale-recovery-candidate-stamped-at-read` | C44 | cache-contract.qnt | `retainedCandidateWasEligible` | mapped `M44` | `compressedRecoveryRechecksMaximumAfterDecodeTest` | 3 / 8 | | `shadow-fenced-fill-writes` | C53 | kernel/remote-writes.qnt | `fillsClearTheirFence` | mapped `M04` (cross-contract) | `fenceRejudgedAtDumpReleaseAfterRollbackTest` | 3 / 3 | | `shadow-fill-before-source` | C49 | kernel/shadow.qnt | `fillsRequireMissAndAcceptedSource` | mapped `M47` | `darkReadBeforeSourceWaitsForItTest` | 3 / 2 | +| `shadow-logging-true-shorthand-expands-off` | C60 | kernel/mismatch-logging.qnt | `capturedLoggingFollowsLayers` | mapped `M65` | `runtimeTrueShorthandOverridesDefaultValueTest` | 1 / 0 | +| `shadow-logging-malformed-shorthand-expands-on` | C60 | kernel/mismatch-logging.qnt | `capturedLoggingFollowsLayers` | mapped `M08` | `invalidLoggingKeepsComparisonWithoutWarningTest` | 1 / 0 | +| `shadow-logging-written-off-leaf-ignored` | C60 | kernel/mismatch-logging.qnt | `capturedLoggingFollowsLayers` | mapped `M71` | `runtimeOffDiffOverridesDefaultDiffTest` | 1 / 0 | +| `shadow-logging-omitted-value-resolves-on` | C60 | kernel/mismatch-logging.qnt | `capturedLoggingFollowsLayers` | mapped `M09` | `omittedLoggingKeepsMismatchMetricsOnlyTest` | 1 / 0 | +| `shadow-logging-invalid-diff-leaf-acts-values` | C60 | kernel/mismatch-logging.qnt | `capturedLoggingFollowsLayers` | mapped `M76` | `invalidDiffLeafKeepsValueLeafTest` | 1 / 0 | +| `shadow-warning-with-logging-off` | C60 | kernel/shadow.qnt | `noWarningWithLoggingOff` | mapped `M74` | `runtimeFalseShorthandOverridesDefaultDiffTest` | 2 / 3 | +| `shadow-diff-hook-failure-falls-back-to-walk` | C60 | kernel/shadow.qnt | `warningFieldsMatchCapturedLeaves` | mapped `M69` | `hookErrorNullsDiffFieldsTest` | 1 / 4 | +| `shadow-diff-hook-runs-at-diff-off` | C60 | kernel/shadow.qnt | `loggingOutcomePrecedence` | mapped `M70` | `hookSkippedWithDiffOffTest` | 1 / 4 | +| `shadow-warning-paths-carry-entries` | C60 | kernel/shadow.qnt | `warningFieldsMatchCapturedLeaves` | mapped `M75` | `valueWithDiffPathsLogsRenderingsAndPathsTest` | 1 / 4 | +| `shadow-logging-render-error-outranks-hook-error` | C60 | kernel/shadow.qnt | `loggingOutcomePrecedence` | mapped `M72` | `hookErrorOutranksRenderErrorTest` | 1 / 4 | +| `shadow-logging-logger-error-outranks-construction` | C60 | kernel/shadow.qnt | `loggingOutcomePrecedence` | mapped `M73` | `hookErrorOutranksLoggerErrorTest` | 1 / 4 | | `redis-protocol-inclusive-fence` | C33 | cache-rules.qnt | `fenceCanPrecedeEncodingError` | mapped `M06` | `frameAtWatermarkIsDeclinedTest` | 4 / 12 | | `redis-protocol-untracked-fence` | W04 | cache-rules.qnt | `untrackedDecodingIgnoresWatermarks` | mapped `M52` | `untrackedReadsIgnoreGroupedInvalidationTest` | 2 / 14 | | `frame-vectors-inclusive-fence` | C33 | dialcache-frame-vectors.qnt | `trackedHitsStrictlyClearValidFence` | mapped `M06` | `frameAtWatermarkIsDeclinedTest` | 0 / 0 | diff --git a/formal/generated/MUTANTS.md b/formal/generated/MUTANTS.md index 3f36dddf..baa2f435 100644 --- a/formal/generated/MUTANTS.md +++ b/formal/generated/MUTANTS.md @@ -18,8 +18,8 @@ per port, detections, the challenges mapped to it); | `M05` | C03.detached-bypass | Treat a detached context whose outer scope closed as still enabled | ordinary, generated, portable | generated, portable | generated, fixed, portable | none | | `M06` | W04.strict-fence | Accept a tracked frame timestamp equal to its watermark | ordinary, generated, portable | generated, portable | generated, fixed, portable | `fence-inclusive-timestamp`, `tracked-read-inclusive-fence`, `redis-protocol-inclusive-fence`, `frame-vectors-inclusive-fence` | | `M07` | C51.changed-confirmation | Confirm an unequal shadow result even when C1 contains different payload bytes | ordinary, generated, portable | generated, portable | generated, fixed, portable | none | -| `M08` | C60.invalid-logging-policy | Enable mismatch logging after a malformed runtime logging flag | ordinary, generated, portable | generated, portable | generated, fixed, portable | none | -| `M09` | C60.logging-default-off | Enable mismatch logging when its flag is omitted | ordinary, generated, portable | generated, portable | generated, portable | none | +| `M08` | C60.invalid-logging-policy | Enable mismatch logging after a malformed runtime logging flag | ordinary, generated, portable | generated, portable | generated, fixed, portable | `shadow-logging-malformed-shorthand-expands-on` | +| `M09` | C60.logging-default-off | Enable mismatch logging when its flag is omitted | ordinary, generated, portable | generated, portable | generated, portable | `shadow-logging-omitted-value-resolves-on` | | `M10` | C31.tracked-no-local-fill | Publish an unvalidated tracked source value directly into local storage | ordinary, generated, portable | generated, portable | generated, fixed, portable | `core-tracked-fallback-warms-local` | | `M11` | C09.fixed-local-ttl | Renew the local insertion TTL on every cache hit | ordinary, generated, portable | generated, portable | generated, fixed, portable | `local-clock-hit-renews-insertion` | | `M12` | W02.sorted-arguments | Reverse the prescribed argument-name ordering | generated, portable | generated, portable | generated, fixed, portable | none | @@ -74,3 +74,15 @@ per port, detections, the challenges mapped to it); | `M62` | C54.read-ownership | Release shadow capacity at bounded read failure before the raw adapter operation settles | generated, portable | generated, portable | - | `shadow-read-deadline-releases-raw-capacity` | | `M63` | C38.marker-transition | Move an invalidation cutoff backwards when an older request arrives | generated, portable | generated, portable | - | `invalidation-transition-cutoff-moves-backwards` | | `M64` | C39.numeric-limits | Reject the maximum supported invalidation buffer | generated, portable | generated, portable | - | `invalidation-transition-inclusive-buffer-limit` | +| `M65` | C60.logging-enabled | Leave mismatch logging off when its flag is true | ordinary, generated, portable | generated, portable | generated, fixed, portable | `shadow-logging-true-shorthand-expands-off` | +| `M66` | W10.mismatch-paths | Walk arrays of different length over their common prefix instead of reporting them once at the array pointer | ordinary, portable | fixed, portable | fixed, portable | none | +| `M67` | W10.mismatch-paths | Report nothing for a pair of values of different JSON kinds, roots included | ordinary, portable | fixed, portable | fixed, portable | none | +| `M68` | W10.mismatch-paths | Treat an absent compared value as a value with no JSON rendering | ordinary, portable | fixed, portable | fixed, portable | none | +| `M69` | C60.logging-outcomes | Fall back to the built-in walk when the mismatch diff hook fails | ordinary, generated, portable | generated, portable | generated, portable | `shadow-diff-hook-failure-falls-back-to-walk` | +| `M70` | C60.outcome-precedence | Run the mismatch diff hook when diff is off | ordinary, generated, portable | generated, portable | generated, portable | `shadow-diff-hook-runs-at-diff-off` | +| `M71` | C60.shorthand-table | Treat a mismatch-logging leaf written off as unwritten | ordinary, generated, portable | generated, portable | generated, portable | `shadow-logging-written-off-leaf-ignored` | +| `M72` | C60.outcome-precedence | Report render_error over hook_error when both fail | ordinary, generated, portable | generated, portable | generated, portable | `shadow-logging-render-error-outranks-hook-error` | +| `M73` | C60.outcome-precedence | Report logger_error over a warning construction failure when the logger also fails | ordinary, generated, portable | generated, portable | generated, portable | `shadow-logging-logger-error-outranks-construction` | +| `M74` | C60.logging-outcomes | Emit a mismatch warning when both mismatch-logging leaves are off | ordinary, generated, portable | generated, portable | generated, fixed, portable | `shadow-warning-with-logging-off` | +| `M75` | C60.warning-field-groups | Log the difference entries with their values at diff paths | ordinary, generated, portable | generated, portable | generated, fixed, portable | `shadow-warning-paths-carry-entries` | +| `M76` | C60.invalid-input-isolation | Resolve an invalid diff leaf to values instead of off | ordinary, generated, portable | generated, portable | generated, portable | `shadow-logging-invalid-diff-leaf-acts-values` | diff --git a/formal/generated/PROFILES.md b/formal/generated/PROFILES.md index 283dafed..5c4d08d9 100644 --- a/formal/generated/PROFILES.md +++ b/formal/generated/PROFILES.md @@ -20,7 +20,7 @@ fields and the observation record; [PORTING.md](../guides/PORTING.md) explains w | [scope](#scope) | [dialcache-scope-conformance.qnt](../models/dialcache-scope-conformance.qnt) | 7 | 256 | 14 | 26 | | [recovery](#recovery) | [dialcache-recovery-conformance.qnt](../models/dialcache-recovery-conformance.qnt) | 14 | 512 | 31 | 47 | | [policy](#policy) | [dialcache-policy-conformance.qnt](../models/dialcache-policy-conformance.qnt) | 12 | 512 | 33 | 51 | -| [shadow](#shadow) | [dialcache-shadow-conformance.qnt](../models/dialcache-shadow-conformance.qnt) | 20 | 1024 | 40 | 77 | +| [shadow](#shadow) | [dialcache-shadow-conformance.qnt](../models/dialcache-shadow-conformance.qnt) | 21 | 1024 | 61 | 115 | | [admission](#admission) | [dialcache-admission-conformance.qnt](../models/dialcache-admission-conformance.qnt) | 8 | 128 | 9 | 19 | | [layers](#layers) | [dialcache-layers-conformance.qnt](../models/dialcache-layers-conformance.qnt) | 8 | 512 | 17 | 27 | | [independent](#independent) | [dialcache-independent-conformance.qnt](../models/dialcache-independent-conformance.qnt) | 11 | 512 | 15 | 18 | @@ -266,29 +266,32 @@ The coordinator sends an empty fixture; see [core.mjs](../replay/core.mjs) for w | | | | --- | --- | | Model | [formal/models/dialcache-shadow-conformance.qnt](../models/dialcache-shadow-conformance.qnt) | -| Version | 3, inputs `explicit-v1` | +| Version | 4, inputs `explicit-v1` | | Smoke trace | [shadow-smoke.itf.json](./shadow-smoke.itf.json) | | Ports claiming it | typescript, go, rust, python | -| Generation | 1024 sampled histories of at most 60 steps from 4096 samples; 14 scheduled invariants; 40 exported regressions | -| Required witnesses | 77 labels in [coverage-witnesses.json](../catalogs/coverage-witnesses.json) | +| Generation | 1024 sampled histories of at most 60 steps from 4096 samples; 19 scheduled invariants; 61 exported regressions | +| Required witnesses | 115 labels in [coverage-witnesses.json](../catalogs/coverage-witnesses.json) | ### Fixture, by init choice | Init choice | Fixture | | --- | --- | -| 0 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 1 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"comparator":"equal","observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 2 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"comparator":"unequal","observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 3 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"comparator":"error","observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 4 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"logMismatches":true}},"tracked":true,"shadowHook":true,"observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 5 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"logMismatches":true}},"tracked":true,"shadowHook":true,"comparator":"equal","observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 6 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"logMismatches":true}},"tracked":true,"shadowHook":true,"comparator":"unequal","observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 7 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"logMismatches":true}},"tracked":true,"shadowHook":true,"comparator":"error","observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 8 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":false,"observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 9 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"comparator":"equal","comparisonMs":10,"observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 10 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"comparator":"error","comparisonMs":10,"observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 11 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"sourceWorkMs":9,"observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | -| 12 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"sourceWorkMs":10,"observe":["shadowAge","mismatchWarning","coalesced","error","futureOffset"]}` | +| 0 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 1 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"comparator":"equal","observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 2 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"comparator":"unequal","observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 3 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"comparator":"error","observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 4 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"logMismatches":true}},"tracked":true,"shadowHook":true,"observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 5 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"logMismatches":true}},"tracked":true,"shadowHook":true,"comparator":"equal","observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 6 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"logMismatches":true}},"tracked":true,"shadowHook":true,"comparator":"unequal","observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 7 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"logMismatches":true}},"tracked":true,"shadowHook":true,"comparator":"error","observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 8 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":false,"observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 9 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"comparator":"equal","comparisonMs":10,"observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 10 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"comparator":"error","comparisonMs":10,"observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 11 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"sourceWorkMs":9,"observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 12 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100}},"tracked":true,"shadowHook":true,"sourceWorkMs":10,"observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 13 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"logMismatches":true}},"tracked":true,"shadowHook":true,"mismatchDiff":"error","observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 14 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"mismatchLogging":{"diff":"paths"}}},"tracked":true,"shadowHook":true,"observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | +| 15 | `{"policy":{"ttlSec":{"remote":60},"ramp":{"remote":0},"shadow":{"ramp":100,"mismatchLogging":{"value":false}}},"tracked":true,"shadowHook":true,"observe":["shadowAge","mismatchWarning","shadowMismatchLogging","coalesced","error","futureOffset"]}` | ### Setup commands @@ -299,7 +302,7 @@ The coordinator sends an empty fixture; see [core.mjs](../replay/core.mjs) for w | Action | Choices | Driver commands | | --- | --- | --- | | `beginCall` | none | `{"op":"begin"}` | -| `resolveLoader` | 1, 2 | 1 → `{"op":"resolve","loader":0,"value":1}`
2 → `{"op":"resolve","loader":0,"value":2}`
every decoding reads `observation.loaders` (the count so far at the step; shown with one so far) | +| `resolveLoader` | 1, 2, 12 | 1 → `{"op":"resolve","loader":0,"value":1}`
2 → `{"op":"resolve","loader":0,"value":2}`
12 → `{"op":"resolve","loader":0,"unrenderable":true}`
every decoding reads `observation.loaders` (the count so far at the step; shown with one so far) | | `rejectLoader` | none | `{"op":"reject","loader":0}`
every decoding reads `observation.loaders` (the count so far at the step; shown with one so far) | | `releaseRead` | none | `{"op":"release","effect":"read","index":0}`
every decoding reads `observation.reads` (the count so far at the step; shown with one so far) | | `releaseLoad` | none | `{"op":"release","effect":"load","index":0}`
every decoding reads `observation.loads` (the count so far at the step; shown with one so far) | @@ -314,10 +317,11 @@ The coordinator sends an empty fixture; see [core.mjs](../replay/core.mjs) for w | `loadFault` | 0, 1 | 0 → `{"op":"faults","value":{"load":false}}`
1 → `{"op":"faults","value":{"load":true}}` | | `dumpFault` | 0, 1 | 0 → `{"op":"faults","value":{"dump":false}}`
1 → `{"op":"faults","value":{"dump":true}}` | | `writeFault` | 0, 1 | 0 → `{"op":"faults","value":{"write":false}}`
1 → `{"op":"faults","value":{"write":true}}` | +| `loggerFault` | 0, 1 | 0 → `{"op":"faults","value":{"logger":false}}`
1 → `{"op":"faults","value":{"logger":true}}` | | `rollbackWall` | none | `{"op":"shiftWall","ms":-1000}` | | `advanceWall` | 1, 60000 | 1 → `{"op":"shiftWall","ms":1}`
60000 → `{"op":"shiftWall","ms":60000}` | | `shadowPolicy` | 0..2 | 0 → `{"op":"policy","value":{"shadow":{"ramp":100}}}`
1 → `{"op":"policy","value":{"shadow":{"ramp":0}}}`
2 → `{"op":"policy","value":{"shadow":{"ramp":101}}}` | -| `logPolicy` | 0..2 | 0 → `{"op":"policy","value":{"shadow":{"logMismatches":false}}}`
1 → `{"op":"policy","value":{"shadow":{"logMismatches":true}}}`
2 → `{"op":"policy","value":{"shadow":{"logMismatches":"invalid"}}}` | +| `logPolicy` | 0..12 | 0 → `{"op":"policy","value":{"shadow":{"logMismatches":false}}}`
1 → `{"op":"policy","value":{"shadow":{"logMismatches":true}}}`
2 → `{"op":"policy","value":{"shadow":{"logMismatches":"invalid"}}}`
3 → `{"op":"policy","value":{"shadow":{"mismatchLogging":{"diff":"paths"}}}}`
4 → `{"op":"policy","value":{"shadow":{"mismatchLogging":{"diff":"values"}}}}`
5 → `{"op":"policy","value":{"shadow":{"logMismatches":false,"mismatchLogging":{"value":true}}}}`
6 → `{"op":"policy","value":{"shadow":{"logMismatches":true,"mismatchLogging":{"value":false}}}}`
7 → `{"op":"policy","value":{"shadow":{"logMismatches":false,"mismatchLogging":{"diff":"paths"}}}}`
8 → `{"op":"policy","value":{"shadow":{"mismatchLogging":{"value":true,"diff":"invalid"}}}}`
9 → `{"op":"policy","value":{"shadow":{"mismatchLogging":{"value":"invalid","diff":"paths"}}}}`
10 → `{"op":"policy","value":{"shadow":{"logMismatches":"invalid","mismatchLogging":{"diff":"values"}}}}`
11 → `{"op":"policy","value":{"shadow":{"mismatchLogging":{"diff":"off"}}}}`
12 → `{"op":"policy","value":{"shadow":{"mismatchLogging":{"value":false}}}}` | ### Observation channels diff --git a/formal/generated/generated-fixtures.lock.json b/formal/generated/generated-fixtures.lock.json index 3e63853c..480de364 100644 --- a/formal/generated/generated-fixtures.lock.json +++ b/formal/generated/generated-fixtures.lock.json @@ -6,14 +6,14 @@ "seed": "0xd1a1ca" }, "inputs": { - "formal/catalogs/fixture-recipes.json": "afbc6f0d6c9a45364c4c55343eae2cc31a3e266f1606e9b09775c4dc90c37586", + "formal/catalogs/fixture-recipes.json": "0dd657dab379d82a3dbfa8e815afa6d55e7d7c659d9b5c12c8cdda1793d71423", "formal/models/cache-contract.qnt": "fabf5cf46d24d62cee8cbd88ff7233a965b4d038b622b18b92760a9cbd387a35", "formal/models/cache-rules.qnt": "ae2a1748e911402c6818e8294265764e06d18d050a89219df7d55faa83fb06e5", "formal/models/cohort-boundaries.qnt": "700f0b9ee734f2511b3c7f10138b9636cef93bc1045b653479a05be1766e3407", - "formal/models/conformance-observations.qnt": "76e26fb0c5802a13c63f0819071b5326b1586cab187c73d8b204ec7da70f5c2f", - "formal/models/dialcache-admission-conformance.qnt": "08ca435b9f3c200c2386a2ce2da7fd0651bd0f8d24f02083242f3674e1db60de", + "formal/models/conformance-observations.qnt": "10c152ca2cbbf9260bdb6b3f5ae45300208e9357fc3f0261e1b922aed2536b6b", + "formal/models/dialcache-admission-conformance.qnt": "2b52cf4afa96dfcb6622109884f310a2754dca16b20e8a50b69563d36217cf8c", "formal/models/dialcache-conformance.qnt": "3e91cad0afa6ba44353d651078fdf6284df0354ca82de0e3712dfe4f243244d7", - "formal/models/dialcache-dark-layers-conformance.qnt": "e9ffc9133a6a1b641aff31123b48799273d64bd5596f0c504879d3eb085a2e12", + "formal/models/dialcache-dark-layers-conformance.qnt": "0b7aec41fa8989087e744421a73239b2f8376be6a527340e42d4baeebac4a4a7", "formal/models/dialcache-effects-conformance.qnt": "34f7b08029304a0a499219bb8dbc3fc19982c0d5ba9f073cfc806e1c73aba24c", "formal/models/dialcache-independent-conformance.qnt": "948dda299145fd5afd179f2ce6b98c17aa3796a88ff90dc7c44da8f437c1ca50", "formal/models/dialcache-layers-conformance.qnt": "c596ae3e8d4a10d4eb9b696ad8ed93b4d4f7f9bbf234ad7ab29016d411a8538e", @@ -24,9 +24,9 @@ "formal/models/dialcache-recovery-read-conformance.qnt": "199bc6c93d74d228c4df500eda8d838970670cb038ba396838d9ba59690d7434", "formal/models/dialcache-runtime-boundaries-conformance.qnt": "dc761bf9cf65b91382477ead6d2596582089512c0405300e813ab6c909079c96", "formal/models/dialcache-scope-conformance.qnt": "1f8817b882b5f8c4d37aecf82d2b4ae1f87f1dc5fd55bfdd94bf4841b6b1fddd", - "formal/models/dialcache-shadow-conformance.qnt": "8fcb59b1012f203f8248cdab095cfc3c25f0860715b1b44c3fc0ffbfed4510a6", - "formal/models/dialcache-shadow-layers-conformance.qnt": "d778c8e4efd2f62e33de17e8739bf3d49d426b8b47e2a129056d8a485db9aff7", - "formal/models/dialcache-shadow-read-deadlines-conformance.qnt": "b43373789a9cbfd907b50c5ebb49dec738e3621eab8132d16a4101634ed0d9f2", + "formal/models/dialcache-shadow-conformance.qnt": "8e2f1ff1af1bc33d20c7c679548e14d13d6deb1e26827b89732fcd12748ee6b5", + "formal/models/dialcache-shadow-layers-conformance.qnt": "1844262e3692cf3e1030deee522ad13abe2e65dbc9e95c92b99d9b30ca927101", + "formal/models/dialcache-shadow-read-deadlines-conformance.qnt": "6956b66b291ddf4eedf1c06f8e42baa4332107d15a0869b5bcfb4e7759f07ae5", "formal/models/dialcache-source-budgets-conformance.qnt": "d2e01cc2cf6caed229e289c727a2bbc7e99b7d9557a9e94223cc399732dd20b1", "formal/models/kernel/adapter-replies.qnt": "00a8a2d28501f037855815dc4468f0a1f8c26eaf346f5b719ff35c1e5e7ba377", "formal/models/kernel/calls.qnt": "914e035dcb91928cb7f70a205308ce545829c81aa91fcb8bee61b2071a71e96f", @@ -44,6 +44,7 @@ "formal/models/kernel/local-storage.qnt": "d977be3273f8431a6b190afb76ba02340ee1926a62fa6da933c976e2286305c7", "formal/models/kernel/markers.qnt": "bde629cc0dcefbf3872a08a16f2f32c727fcfb437d43191f24987cdbc670a323", "formal/models/kernel/metrics.qnt": "66b105998eb3aaba9ea473138544b1b5b8d7608a97f8f72737dd44799177210f", + "formal/models/kernel/mismatch-logging.qnt": "804df021443164923bfe526747ed6366a9827d78bed0d0878a93d0bc060e4174", "formal/models/kernel/payloads.qnt": "1ac3a418e946ac018cd1945ff0df22297d2357e3e371e009aa7572f3a38badd4", "formal/models/kernel/policy-gate.qnt": "9f4a5fa19d22c96a987eaeb01d3d9442fce276ddd88c15af8dbf2aae81fb1885", "formal/models/kernel/policy-overlay.qnt": "00986037b03ea858bc99ffe13ea46925e2f0a8707950fa2204d15c93f6e0fb90", @@ -56,7 +57,7 @@ "formal/models/kernel/runtime-policy.qnt": "af91c15b1883b15cc84f28a36275d49bca2dd183c926dbe5225632c8cf913fe6", "formal/models/kernel/serving.qnt": "120a879170889db5b1078da56dac6b27e69fe1b593691162bc3ecfa03cac7784", "formal/models/kernel/shadow-reads.qnt": "f1626ed16a62454b36a77ec644115ce02789650468c8642a010479d76d546b1f", - "formal/models/kernel/shadow.qnt": "cb40b05105cab3e63695f581a9749b3579d42d6267982617dcee39efe5b69c9f", + "formal/models/kernel/shadow.qnt": "4b8b0af63424a008cbd4f7c10edc9561e40ca14ac65c7c9b2014443a53bf6286", "formal/tools/compact-json.mjs": "fe180d42ea438f915a011accf79728701fd51222877d39adbda8ea3ebf3f4bfb", "formal/tools/generated-fixtures.mjs": "2f1938d6b3b6f97633dbf0277d5e014590110fc9da1e6817a4d3943e68243392" }, @@ -67,7 +68,7 @@ "formal/generated/recovery-smoke.itf.json": "1fa6a1cc49fa1c47c385c681a9961e81422bb0f8bcc343ae9ca3b4b3bcdc14ee", "formal/generated/policy-smoke.itf.json": "fd1cd858d73726fdd2cc10c26d1b9490ba3ffd1f0f64f10dfef4047420022004", "typescript/test/fixtures/policy-shadow-witnesses.json": "fc962fd07e54dcae7ff08f4a473e58dbc4a462e1b4959b1744cd70c84fb55f2e", - "formal/generated/shadow-smoke.itf.json": "bed7e93679a0e02f1a9f126830ba9f14696d72c767dd80445c071a8dd3359b6f", + "formal/generated/shadow-smoke.itf.json": "4d7ad407d9f2d8cbb3f31885342213d431825d23e3ed56bc3afc8553493b583c", "formal/generated/admission-smoke.itf.json": "0f7ddbdb13e2b61f824cf12013fc4848b04334ba58c6fac51434fefce5577d6e", "formal/generated/layers-smoke.itf.json": "82a291e920d048121f0351ba1c72a2c9d3f843991d67581920240c854c7947b2", "formal/generated/independent-smoke.itf.json": "d978317b833afcb1b7714fff6c18558b793558620b8ea0733853ae2ba08ca88a", @@ -80,12 +81,12 @@ "typescript/test/fixtures/effects-authority-witnesses.json": "9f3b2b5c3c3093d5dae40c7b7c5621a9110f18accfd7b6747f18f3a94c066bc2", "typescript/test/fixtures/formal-runtime-boundary-witnesses.json": "e8b44a1821b1053f4d40e5e217be48105c7de588032dcda55bb690c92370564e", "typescript/test/fixtures/formal-runtime-witness-boundaries.json": "48d0cdd94c4763cda9ada1005337bfd0ab8759f7fbaf1726c0213b2b96aef3c0", - "typescript/test/fixtures/formal-witness-attribution.json": "6b07eb779d85e6a5b85487f4ddeebf316c19961f16470f1916be6361b134961b", - "typescript/test/fixtures/formal-witness-boundaries.json": "92d73399b3cf0228160aa28ce28c6efac44ebad046015eae8ebc7aedd220cb28", + "typescript/test/fixtures/formal-witness-attribution.json": "ec2f9ee0fb7a86d40188129aa54c2e8189b0e710f39cc6550c8aa611478bf4ab", + "typescript/test/fixtures/formal-witness-boundaries.json": "f4afff886fa3ec7ec9b35457da28f23e3a77111be8112903a65c7695fd106255", "typescript/test/fixtures/local-failure-witnesses.json": "fcf62b450d17edf4429870d6d9bc4d5727199b7fa7a6a4cbf8030546e06b962b", "typescript/test/fixtures/recovery-admission-witnesses.json": "716f0e2f79497b05edaf4e8efaf03e9a762a4b7e0f47fd1aa87c3f1b126b9736", "typescript/test/fixtures/recovery-read-witnesses.json": "522601141e82571a3f0cdd84607247a576eea9ccb24491cd2b89a7073ee7bdac", - "typescript/test/fixtures/shadow-diagnostics-witnesses.json": "02704e5ffe27891511ebb6cba1692bec7620d5b1e6b3e03b0ab3308b9bf81295", + "typescript/test/fixtures/shadow-diagnostics-witnesses.json": "a7bd22481f8f8a685c16d779f4923e896d576001406ff976b86a3f3d5216179b", "typescript/test/fixtures/shadow-layers-witnesses.json": "93db73aa85c9772450f12396b6903d1d09acfd5a5638373fd09654e2e0351813", "typescript/test/fixtures/source-budgets-witnesses.json": "7eba7bd5b61abc4b1e7b69f8b90b1a93f1b4ceb553ddce9355973daa38168b5c", "typescript/test/fixtures/policy-witnesses.json": "ddd1bb653a0c8c1bba3e27e34c0f81516fae45091fe09e23d6f547e5748dc135", diff --git a/formal/generated/shadow-smoke.itf.json b/formal/generated/shadow-smoke.itf.json index 2ba61c23..3e97ca08 100644 --- a/formal/generated/shadow-smoke.itf.json +++ b/formal/generated/shadow-smoke.itf.json @@ -11,66 +11,66 @@ "mbt::nondetPicks" ], "states": [ - {"input":{"choice":{"#bigint":"0"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"20"},"name":"invalidate"},"mbt::actionTaken":"invalidate","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"20"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"10"},"name":"advance"},"mbt::actionTaken":"advance","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"10"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"0"},"name":"shadowPolicy"},"mbt::actionTaken":"shadowPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"shadowPolicy"},"mbt::actionTaken":"shadowPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseDump"},"mbt::actionTaken":"releaseDump","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"writeFault"},"mbt::actionTaken":"writeFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"2"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"0"},"name":"loadFault"},"mbt::actionTaken":"loadFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"rejectLoader"},"mbt::actionTaken":"rejectLoader","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"6"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"6"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"3"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"3"},"reads":{"#bigint":"3"},"recovery":[],"shadow":["timeout","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"2"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"rejectLoader"},"mbt::actionTaken":"rejectLoader","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"3"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"3"},"reads":{"#bigint":"3"},"recovery":[],"shadow":["timeout","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"2"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"3"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"3"},"reads":{"#bigint":"3"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"2"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"4"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"4"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"3"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"3"},"reads":{"#bigint":"3"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"2"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"writeFault"},"mbt::actionTaken":"writeFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"8"},"name":"seedUnicode"},"mbt::actionTaken":"seedUnicode","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"8"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"7"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"7"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"advance"},"mbt::actionTaken":"advance","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"dumpFault"},"mbt::actionTaken":"dumpFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"5"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"5"},"reads":{"#bigint":"5"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"4"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"0"},"name":"readFault"},"mbt::actionTaken":"readFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"5"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"5"},"reads":{"#bigint":"5"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"4"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"5"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"5"},"reads":{"#bigint":"5"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"4"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"rejectLoader"},"mbt::actionTaken":"rejectLoader","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"5"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"5"},"reads":{"#bigint":"5"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"4"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"6"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"6"},"reads":{"#bigint":"6"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"5"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"6"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"6"},"reads":{"#bigint":"6"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"5"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"6"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"6"},"reads":{"#bigint":"6"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"5"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"7"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"7"},"reads":{"#bigint":"7"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"6"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"7"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"7"},"reads":{"#bigint":"7"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"6"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"rejectLoader"},"mbt::actionTaken":"rejectLoader","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"7"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"7"},"reads":{"#bigint":"7"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"6"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"advance"},"mbt::actionTaken":"advance","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"7"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"7"},"reads":{"#bigint":"7"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"6"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"10"},"name":"advance"},"mbt::actionTaken":"advance","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"10"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"7"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"7"},"reads":{"#bigint":"7"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"6"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"8"},"name":"seedUnicode"},"mbt::actionTaken":"seedUnicode","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"8"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"20"},"name":"invalidate"},"mbt::actionTaken":"invalidate","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"20"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"0"},"name":"loadFault"},"mbt::actionTaken":"loadFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"8"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"8"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"0"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"2"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"20"},"name":"invalidate"},"mbt::actionTaken":"invalidate","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"20"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"readFault"},"mbt::actionTaken":"readFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"0"},"name":"shadowPolicy"},"mbt::actionTaken":"shadowPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"0"},"name":"loadFault"},"mbt::actionTaken":"loadFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"rollbackWall"},"mbt::actionTaken":"rollbackWall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"shadowPolicy"},"mbt::actionTaken":"shadowPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"0"},"name":"readFault"},"mbt::actionTaken":"readFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"0"},"name":"dumpFault"},"mbt::actionTaken":"dumpFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}} + {"input":{"choice":{"#bigint":"0"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"20"},"name":"invalidate"},"mbt::actionTaken":"invalidate","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"20"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"10"},"name":"advance"},"mbt::actionTaken":"advance","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"10"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"shadowPolicy"},"mbt::actionTaken":"shadowPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"shadowPolicy"},"mbt::actionTaken":"shadowPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseDump"},"mbt::actionTaken":"releaseDump","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"writeFault"},"mbt::actionTaken":"writeFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"loadFault"},"mbt::actionTaken":"loadFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"rejectLoader"},"mbt::actionTaken":"rejectLoader","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"6"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"6"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"2"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"2"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["timeout","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"3"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"3"},"reads":{"#bigint":"3"},"recovery":[],"shadow":["timeout","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"2"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"rejectLoader"},"mbt::actionTaken":"rejectLoader","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"3"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"3"},"reads":{"#bigint":"3"},"recovery":[],"shadow":["timeout","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"2"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"3"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"3"},"reads":{"#bigint":"3"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"2"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"4"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"4"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"3"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"3"},"reads":{"#bigint":"3"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"2"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"writeFault"},"mbt::actionTaken":"writeFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"8"},"name":"seedUnicode"},"mbt::actionTaken":"seedUnicode","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"8"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"7"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"7"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"advance"},"mbt::actionTaken":"advance","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"dumpFault"},"mbt::actionTaken":"dumpFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"4"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"4"},"reads":{"#bigint":"4"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"3"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"5"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"5"},"reads":{"#bigint":"5"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"4"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"readFault"},"mbt::actionTaken":"readFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"5"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"5"},"reads":{"#bigint":"5"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"4"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"5"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"5"},"reads":{"#bigint":"5"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"4"},"fallbackErrors":["local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"rejectLoader"},"mbt::actionTaken":"rejectLoader","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"5"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"5"},"reads":{"#bigint":"5"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"4"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"6"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"6"},"reads":{"#bigint":"6"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"5"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"6"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"6"},"reads":{"#bigint":"6"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"5"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"6"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"6"},"reads":{"#bigint":"6"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"5"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"7"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"7"},"reads":{"#bigint":"7"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"6"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"7"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"7"},"reads":{"#bigint":"7"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"6"},"fallbackErrors":["local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"rejectLoader"},"mbt::actionTaken":"rejectLoader","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"7"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"7"},"reads":{"#bigint":"7"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"6"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"advance"},"mbt::actionTaken":"advance","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"7"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"7"},"reads":{"#bigint":"7"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"6"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"10"},"name":"advance"},"mbt::actionTaken":"advance","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"10"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"7"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"7"},"reads":{"#bigint":"7"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"6"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"8"},"name":"seedUnicode"},"mbt::actionTaken":"seedUnicode","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"8"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"1"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"20"},"name":"invalidate"},"mbt::actionTaken":"invalidate","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"20"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"loadFault"},"mbt::actionTaken":"loadFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"0"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"8"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"8"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"8"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"2"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"20"},"name":"invalidate"},"mbt::actionTaken":"invalidate","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"20"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"readFault"},"mbt::actionTaken":"readFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"shadowPolicy"},"mbt::actionTaken":"shadowPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"loadFault"},"mbt::actionTaken":"loadFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"rollbackWall"},"mbt::actionTaken":"rollbackWall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"shadowPolicy"},"mbt::actionTaken":"shadowPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"readFault"},"mbt::actionTaken":"readFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"dumpFault"},"mbt::actionTaken":"dumpFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"3"},{"#bigint":"1"},{"#bigint":"3"},{"#bigint":"2"},{"#bigint":"3"},{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"3"},"loaders":{"#bigint":"8"},"loads":{"#bigint":"1"},"maintenance":["ok","mutation_error","mutation_error"],"policyCalls":{"#bigint":"8"},"reads":{"#bigint":"9"},"recovery":[],"shadow":["timeout","source_error","source_error","fill_fenced","source_error","fill_fenced","source_error","mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"7"},"fallbackErrors":["local","local","local","local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}} ] } diff --git a/formal/guides/AUTHORING.md b/formal/guides/AUTHORING.md index 6a90c3cd..d05969fe 100644 --- a/formal/guides/AUTHORING.md +++ b/formal/guides/AUTHORING.md @@ -571,7 +571,10 @@ every cohort crashed, so the gate names it while the rest of the shard is measured; a settlement violation under a mutant ([PORTING.md](./PORTING.md)) is recorded the same way, naming the first violating history and rule. New mutants therefore require `generated` and `portable`; require -`ordinary` only where a unit test pins the fault on purpose. To measure one +`ordinary` only where a unit test pins the fault on purpose. A fault only the +checked-in vectors reach, such as the built-in mismatch walk over nested +values, requires `portable` in place of `generated` (with `fixed` in Go and +Rust), and no challenge can map to it. To measure one mutant while authoring it, run `MUTATION_ONLY=M18 make mutations-ts` and `MUTATION_ONLY=M18 make mutations-go`, one partial run per port at a time (they share `partial/`); the partial report under diff --git a/formal/guides/BEHAVIOR.md b/formal/guides/BEHAVIOR.md index d953c7ab..2cbd10df 100644 --- a/formal/guides/BEHAVIOR.md +++ b/formal/guides/BEHAVIOR.md @@ -43,20 +43,20 @@ Every scenario/trace gets a fresh default cache instance and empty Redis environ - `request`, `miss`, `disabled`, `error`, `coalesced`, `invalidation`: public bounded metadata. Operational events use `cacheNamespace`, `useCase`, `keyType`, and `layer` as applicable; miss/disabled events add `reason`, errors add `error`/`inFallback`, and coalescing uses `scope`. - `shadowAge`, `recoveryAge`, `futureOffset`, `get`, `fallback`, `serialization`: actual observations with `seconds`; serialization adds `operation`, and verdict ages add `outcome`. - `size`, `storedSize`: observed `bytes`. `compression` carries its actual `outcome`. -- `mismatchWarning`: fields supplied to the public logger for a confirmed-mismatch warning. The scalar fixtures do not prescribe a general host-language JSON conversion algorithm or truncation implementation. +- `mismatchWarning`: fields supplied to the public logger for a confirmed-mismatch warning, each recorded exactly when the warning carries it; the `mismatchPaths` array is recorded as one JSON string, `mismatchPathsJson`. `shadowMismatchLogging`: the one logging outcome of each warning attempt, with the cache labels and no layer. The shadow profile compares paths, entries and value fields as parsed JSON, so the scalar fixtures do not prescribe a general host-language JSON conversion algorithm or truncation implementation. - `marker`: an explicit environment probe of the tracked watermark's cutoff and remaining physical TTL. Cutoffs are milliseconds relative to the fixture wall origin; absence is `cutoffMs: -1`. TTL uses Redis's `-2` absent and `-1` persistent values. This controlled adapter observation supplements real-server integration evidence. Names are a trace vocabulary, not required language method names. All selected events are compared after every input, including steps that expect none. The [source audit](./TEST-AUDIT.md) explains the typescript/test/doc obligations these probes cover. ## Fixture -`policy` uses the existing configuration vocabulary: `ttlSec` and `ramp` with `local`/`remote` leaves, `requestLocal`, `coalesce`, `staleOnErrorMaxAgeSec`, `remoteReadTimeoutMs`, and `shadow.ramp`. Omitted fields have DialCache's documented defaults; TTL without a ramp enables that layer fully. Runtime overlays initially omit all leaves. +`policy` uses the existing configuration vocabulary: `ttlSec` and `ramp` with `local`/`remote` leaves, `requestLocal`, `coalesce`, `staleOnErrorMaxAgeSec`, `remoteReadTimeoutMs`, `shadow.ramp`, and the mismatch-logging inputs `shadow.logMismatches` and `shadow.mismatchLogging` with its `value` and `diff` leaves. Omitted fields have DialCache's documented defaults; TTL without a ramp enables that layer fully. Runtime overlays initially omit all leaves. -Other fixture fields are `tracked` (default false), `fallbackTimeoutMs` (fixture default 10, null disables, `"default"` omits the operation override to exercise the library's 60-second default), `readTimeoutMs` (fixture default 50; `"default"` omits the adapter option to exercise the library default), `localMaxSize` (default 10,000), `shadowMaxInFlight` (default 1), and `recovery` (`allow`, `deny`, `error`, or default timeout-only classification). An operation can override the instance classifier with `begin.recovery`. Optional `comparator` supplies `equal`, `unequal`, or `error`; omitted comparison uses ordinary value equality. Optional `comparisonMs` makes that external callback consume the specified elapsed time before returning or failing, without delivering timers. These are controlled callback outcomes, not an expression language. `shadowHook: false` omits the required outcome observer; `observerFailure: true` makes installed observers fail; `remote: false` omits the Redis adapter. `probeSourceScope: true` records whether caching is enabled when each actual source invocation begins. Shadow/recovery hooks always record terminal diagnostic outcomes. Optional `observe` selects additional public events (see below); it does not change policy or execution. Compression writes are disabled; existing compressed entries can be seeded to check decoding and recovery. Envelope interoperability is also covered separately by protocol vectors. +Other fixture fields are `tracked` (default false), `fallbackTimeoutMs` (fixture default 10, null disables, `"default"` omits the operation override to exercise the library's 60-second default), `readTimeoutMs` (fixture default 50; `"default"` omits the adapter option to exercise the library default), `localMaxSize` (default 10,000), `shadowMaxInFlight` (default 1), and `recovery` (`allow`, `deny`, `error`, or default timeout-only classification). An operation can override the instance classifier with `begin.recovery`. Optional `comparator` supplies `equal`, `unequal`, or `error`; omitted comparison uses ordinary value equality. Optional `comparisonMs` makes that external callback consume the specified elapsed time before returning or failing, without delivering timers. These are controlled callback outcomes, not an expression language. `shadowHook: false` omits the required outcome observer; `mismatchDiff: "error"` installs a `shadowMismatchDiff` hook that throws; `observerFailure: true` makes installed observers fail, and the `logger` fault makes only the logger fail after it receives a call; `remote: false` omits the Redis adapter. `probeSourceScope: true` records whether caching is enabled when each actual source invocation begins. Shadow/recovery hooks always record terminal diagnostic outcomes. Optional `observe` selects additional public events (see below); it does not change policy or execution. Compression writes are disabled; existing compressed entries can be seeded to check decoding and recovery. Envelope interoperability is also covered separately by protocol vectors. Optional `sourceWorkMs` consumes that many milliseconds of elapsed external work inside each actual source invocation before it returns its unresolved fixture gate. Wall and elapsed clocks advance together; no timers or scheduled cache work run during this segment. This supplies a schedule in which the dark-shadow budget can expire before its deferred job starts, while the source remains independently settleable. Ports provide the same external callback/executor ordering through their controlled environment; neither expected observations nor private cache state select when the job runs. The default is zero work. -Keys use namespace `urn`, key type `id`, use case `Behavior`, and ID `1` unless an input overrides `useCase` or `key`. The fixture value domain is JSON scalars (numbers, strings, booleans, null) plus an absent value. The test serializer uses JSON for scalars and the unquoted literal `undefined` for absence. An omitted input value denotes absence, represented in observations by `{"absent": true}`; the ordinary string `"undefined"` stays a string and cannot be mistaken for it. A port may use an option/unit value or its own fixture sentinel. This custom serializer tests cacheability without prescribing a language's JSON API or reference identity. +Keys use namespace `urn`, key type `id`, use case `Behavior`, and ID `1` unless an input overrides `useCase` or `key`. The fixture value domain is JSON scalars (numbers, strings, booleans, null) plus an absent value, and, for the shadow profile's sources, one value with no JSON rendering (`resolve` with `unrenderable: true`), which a caller observes as `{"unrenderable": true}`. The test serializer uses JSON for scalars and the unquoted literal `undefined` for absence. An omitted input value denotes absence, represented in observations by `{"absent": true}`; the ordinary string `"undefined"` stays a string and cannot be mistaken for it. A port may use an option/unit value or its own fixture sentinel. This custom serializer tests cacheability without prescribing a language's JSON API or reference identity. Wall time starts at `2026-09-08T12:00:00Z`; monotonic time starts at zero. Elapsed time advances through `advance` or configured external source/comparison work; `shiftWall` changes application wall time independently, preserving monotonic time and Redis physical expiry. Local entries use monotonic age; frame age and watermark proposals use application wall time. The fake Redis expiration clock advances with elapsed time independently of application wall-clock steps. Tracked reads atomically acquire a value and watermark. Watermarks remain available through the trace. A native write's frame timestamp is captured when the adapter receives it, before any held transport completion. @@ -134,7 +134,8 @@ trace, the step, the action and both observations. Caller observations encode pending as 0, fixture values 1 and 2 as 1 and 2, a source error as 3 and a deadline error as 4. Codes 5 to 9 are the absent, `null`, `false`, zero and empty-string values the scope and policy profiles generate; 11 is the text -`"undefined"`, distinct from absent; 10 is invalid. Other outcomes fail replay. The +`"undefined"`, distinct from absent; 12 is the shadow profile's source value with no +JSON rendering (D13); 10 is invalid. Other outcomes fail replay. The generated profiles compare error categories; the fixed scenarios compare logical error identity. The effects model keeps the version-2 codes of its own contract, 0 pending, 1 value, 2 source error and 3 timeout; its composed text records the shared @@ -163,7 +164,12 @@ as an intended input-domain expansion. after unit conversion. Recovery records an age only after a successful retained decode; shadow records the original C0 age at a match or confirmed-mismatch verdict, clamped to zero after a wall rollback, and warns only for a confirmed - mismatch whose admitted policy enabled logging. `coalesced` records request or + mismatch whose admission captured either logging leaf on. Shadow also records + `mismatchWarnings`, each warning's content (its age, both value fields as value + codes, -1 left out and 0 null, the diff level whose fields it carries and whether + they are null, the uncut count, the paths, the entries at `values` and the failure + it names), and `loggingOutcomes`, one per warning; dark-layers and + shadow-read-deadlines carry both empty and the parser accepts them absent. `coalesced` records request or process scope labels; `fallbackErrors` records the layer a failed source is attributed to, failures later recovered as stale included. Scope fixtures assert one trail for shared failures and no cache-error trail for disabled pass-through work. @@ -252,10 +258,14 @@ as an intended input-domain expansion. read while the caller stays pending, and the caller's later settlement or timer gives its deadline error; these fixtures observe after overdue timers are delivered, and intermediate native timer ordering is outside their bounds. Timer - delivery is not required for elapsed-time rejection. Malformed logging keeps - comparison active with warnings disabled, and a job admitted under it records a - configuration error and captures logging off; later runtime changes do not rewrite - an admitted job's policy. + delivery is not required for elapsed-time rejection. A malformed logging input + keeps comparison active with that leaf off, a valid leaf beside it on, and a job + admitted under it records one configuration error; later runtime changes do not + rewrite an admitted job's leaves. Fixture 13 installs the throwing diff hook, + fixtures 14 and 15 write default leaves, `logPolicy` writes the overlays of the + shorthand table, the invalid inputs and each leaf written off, `loggerFault` fails + the logger, and the source value with no rendering is offered only while no job + could fill with it, so every failure outcome is reached alone and combined. - **admission.** Caller reads and all decoding are held; a successful read acquires C0 and completing its decode considers shadow admission under the flight's accepted policy; coalesced followers share one admission; a detached source runs with caching diff --git a/formal/guides/CONTRACTS.md b/formal/guides/CONTRACTS.md index ce102866..c869e4b2 100644 --- a/formal/guides/CONTRACTS.md +++ b/formal/guides/CONTRACTS.md @@ -113,7 +113,7 @@ Exporter compatibility and resource-exhaustion limits have explicit native integ | C57 | Recovery age is measured after successful decode only; shadow age uses original C0 at match/confirmed-mismatch verdict, clamped at zero; future offsets are positive and attributed to the observing layer | S: recovery age/negative-outcome scenarios, shadow verdict age scenarios, serving/dark/confirmation future-offset scenarios; G: recovery/shadow (age at decode/verdict and rollback clamping), effects (serving future-offset attribution) | | C58 | Diagnostic categories describe the failed operation and reached layer; coalesced followers do not duplicate a leader's trail; recovered failures still count as source failures; sizes count bytes and precede dispatch | S: stable diagnostic category scenarios, `coalesced remote failure records one leader trail and one follower event`, `request followers report request scope without repeating lower traversal`, `recovered source failure remains a fallback diagnostic`, `source timeout records one failure even after a late rejection`, `accepted publication time is separate from reported source duration`; G: effects (ordered categories, shared trail, phase ownership and pre-dispatch sizes), scope/recovery/shadow (coalescing and source-failure attribution); native local-fault injection is replayed under C27, exporter schemas remain X01 | | C59 | Ordinary remote-get duration includes fresh decode; source duration ends at accepted settlement/deadline and excludes later preparation/write; serialization duration measures its own phase | S: `remote get duration includes fresh decoding without spending a source budget`, `accepted publication time is separate from reported source duration`; G: effects; these assert controlled elapsed time, not real-world timer precision or scheduler turns | -| C60 | Mismatch logging defaults off; opting in emits one diagnostic warning only after confirmed mismatch, independently of caller return and outcome metrics | S: four `mismatch warning … verdict` scenarios; G: shadow (logging default, both policies and admission snapshot); native JSON conversion, exact text truncation, and logger backend remain B03/X01 | +| C60 | Mismatch logging defaults off; opting in, with `value` true or `diff` above `off` once each layer's `logMismatches` expands, emits one diagnostic warning only after confirmed mismatch, independently of caller return and outcome metrics, carrying the field groups the leaves captured at admission name; each warning attempt reports one `shadowMismatchLogging` outcome and logging off reports none, `hook_error` over `render_error` over `logger_error`, a failure nulling only the fields it feeds | S: four `mismatch warning … verdict` scenarios; G: shadow (logging default, each leaf, the shorthand table, invalid inputs, the admission snapshot, warning content, and each logging outcome alone and combined); native JSON conversion, exact value-field and `diffJson` truncation text at their 8 KiB caps, and logger backend remain B03/X01 | ### Deterministic protocol obligations @@ -128,6 +128,7 @@ Exporter compatibility and resource-exhaustion limits have explicit native integ | W07 | Compression threshold counts serialized bytes; write compression must shrink stored data and preserve decoded type/value | V: `compressionWriteVectors`; compressor output bytes are not prescribed | | W08 | Disabling new compression still requires envelope escaping and decompression of old entries | V: `envelopeVectors`, `compressedDecodeVectors`; behavioral fixtures decode through real DialCache with new compression disabled | | W09 | Invalidation marker domain, monotonic cutoff, repair, persistence, retention floor/slack, and pre-mutation validation | I: all vectors; actual exported protocol runs on Redis 6.2 and Valkey 8 | +| W10 | The built-in mismatch walk over two JSON values: RFC 6901 pointers, object members in UTF-16 code-unit order with a one-sided member at its pointer, equal-length arrays by index and others once at the array, roots of different kinds or an absent side (D13) at the root, identical values with no entry (numbers compare as IEEE doubles; a bool never equals a number), depth capped at 32 tokens, entries cut at 64 with the uncut count, pointers clamped at 512 bytes and sides over 1 KiB as clamped text | V: `mismatchPathVectors`; shadow model scalar cases (no entry, or one root entry) | ## Assumptions and deliberate exclusions @@ -157,10 +158,10 @@ Each row identifies where the source's semantic rules land. Repeated tests in di | `docs/coalescing.md` | C11–C15, C18, C23–C28, C36; B02, E03/E04, X01 | | `docs/invalidation.md` | C31–C39, W01/W04/W05/W09; E01–E05 | | `docs/stale-on-error.md` | C37, C40–C46; B01/B02, E04, X01 | -| `docs/shadow-validation.md` | C47–C54; B02/B03, E03/E04, X01 | +| `docs/shadow-validation.md` | C47–C54/C60, W10; B02/B03, E03/E04, X01 | | `docs/redis.md` | C22/C23/C27–C29, W01–W09; B01–B03, E01–E05, X02 | | `docs/api.md` | C01–C54/W01–W09 where those contracts are repeated; B01/B03, X01 for exact API/export/reference contracts | -| `docs/observability.md` | C30/C47/C50–C54, W04; X01 for detailed telemetry contracts | +| `docs/observability.md` | C30/C47/C50–C54/C60, W04; X01 for detailed telemetry contracts | | `docs/upgrading.md`, `docs/maintainers.md` | E05, B01, X01/X02 for deployment, package/release/test workflow, and benchmarks | | Implementation tests (`typescript/test/` basenames) | Inventory | @@ -169,15 +170,15 @@ Each row identifies where the source's semantic rules land. Repeated tests in di | `dialcache-get-or-load` | C01–C05, C11/C15/C18/C27/C31; B01/B03 for inline/wrapper API differences | | `dialcache-coalescing` | C11–C15/C18/C20/C24/C36/C43; B02/B03 | | `dialcache-liveness`, `dialcache-redis-read-deadline` | C23–C28/C46/C56/C59; B01/B02, E03, X01 | -| `dialcache-config-ramp`, `dialcache-observability-internals` | C16–C22/C47, W03; B01, X01 | +| `dialcache-config-ramp`, `dialcache-observability-internals` | C16–C22/C47/C60, W03; B01, X01 | | `dialcache-invalidation` | C29–C39, W01/W09; B01, E01/E02/E05, X01 | | `dialcache-redis` | C05/C22/C27–C29/C31/C33/C34/C55, W04; B01–B03, E01/E04, X01 | | `dialcache-stale-on-error`, `dialcache-stale-recovery-policy` | C37/C40–C46; B01–B03, X01 | -| `dialcache-shadow-validation`, `dialcache-shadow-confirmation` | C30/C47–C54; B01–B03, X01 | +| `dialcache-shadow-validation`, `dialcache-shadow-confirmation` | C30/C47–C54/C60; B01–B03, X01 | | `redis-payload`, `duration` | W04/W05/W09; B01/B02 for native reply shapes, rounding representation, exception identity, and zero-copy views | | `compression`, `compression-error`, `compression-guard`, `dialcache-compression` | C06/C27/C34, W06–W08; B01/B03, X01/X02 | | `dialcache-logger`, `dialcache-metrics` | C20/C21/C27–C30/C33/C40–C60; B02, X01 | -| `datadog`, `prometheus`, `shadow-log-json` | C30; B01–B03, X01 | +| `datadog`, `prometheus`, `shadow-log-json` | C30/C60, W10; B01–B03, X01 | | `node-redis`, `valkey-glide` | W04/W05/W09, E01–E03; B01/B02 and X02 for client-specific routing, retry, module identity, error decoration, and lifecycle | | `redis-real.integration`, `redis-cluster.integration` | E01/E02, W01/W04–W09, C22/C27–C39; X02; portable invalidation vectors execute here | | `formal-behavior`, `formal-conformance`, `formal-effects`, `formal-features`, `formal-protocol-vectors` | Execution of the portable artifacts and independent-observation/parser regression checks | diff --git a/formal/guides/FEATURE-COVERAGE.md b/formal/guides/FEATURE-COVERAGE.md index ab8f3128..1818a2bc 100644 --- a/formal/guides/FEATURE-COVERAGE.md +++ b/formal/guides/FEATURE-COVERAGE.md @@ -57,7 +57,7 @@ every family to exact case IDs, native tests, scope notes and assumptions. | Stale recovery | Fresh limit F and exclusive maximum M; future/fenced candidates; success and classifier denial skipping recovery decode; instance/operation/default classifier precedence; own versus propagated timeout; original-error preservation; age checks before and after held decode; retained bytes across invalidation/replacement/expiry; rollback; memo publication; independent candidates and late source settlement. | | Shadow admission, validation and fills | Hook/capacity/cohort prerequisites; served hits versus dark calls; ordinary misses and recovered results; independent dark caller sources; C0/source order; diagnostic match/mismatch/error; C1 payload and fence checks; fill authority and failures; captured logging; deduplication; per-instance capacity; held read/source/decode/dump/write work and timeout ownership. | | Keys, codecs, compression and Redis integration | All key dimensions and argument ordering; UTF-16 cohort hashing; complete frame bytes; malformed text; safe numeric limits; raw/escaped/compressed markers; compression threshold/type/size choice; native value domains; real command dispatch and primary routing; cancellation/retry bindings; bidirectional TypeScript/Go storage. | -| Diagnostics and exporters | Error category and count with caller outcome; leader/follower trails; late rejection counted once; fallback duration excluding publication; remote duration including decode; recovery/shadow ages and future offsets; byte/compression telemetry; warning eligibility; coalescing inspection; exporter schema, privacy and registry ownership. | +| Diagnostics and exporters | Error category and count with caller outcome; leader/follower trails; late rejection counted once; fallback duration excluding publication; remote duration including decode; recovery/shadow ages and future offsets; byte/compression telemetry; warning eligibility, captured logging leaves and the shorthand, warning field groups, the mismatch walk's paths and entries, and each logging outcome with its precedence; coalescing inspection; exporter schema, privacy and registry ownership. | ## How a corner earns evidence diff --git a/formal/guides/GO-PARITY.md b/formal/guides/GO-PARITY.md index 4d01bbb9..91d123ad 100644 --- a/formal/guides/GO-PARITY.md +++ b/formal/guides/GO-PARITY.md @@ -212,11 +212,15 @@ portable case's Quint evidence from its additional native binding requirements. `MetricsAdapter.ObserveEvent` accepts backend-neutral diagnostics. `FailureIsolatedObserver` and `FailureIsolatedLogger` prevent exporter errors -or panics from replacing cache/source results. Shadow mismatch previews use -native JSON behavior and bounded UTF-8 output; unavailable previews remain -absent. The Prometheus and DogStatsD adapters preserve metric names, units, -labels, bucket boundaries, and counter increments. Logical cache keys never -become metric labels. +or panics from replacing cache/source results. Confirmed-mismatch warning fields +render each compared value once with native JSON behavior and bounded UTF-8 +output; the built-in mismatch-path walk runs over the parsed renderings, +`Operation.MismatchDiff` replaces it for one operation, and a field whose +rendering or computation fails is nil. A `Warn` panic stays isolated, and the +`shadowMismatchLogging` event reports it as `logger_error` unless a hook or +render failure outranks it. The Prometheus and DogStatsD adapters preserve +metric names, units, labels, bucket boundaries, and counter increments. Logical +cache keys never become metric labels. The Go Prometheus binding accepts an explicit native registry. Reconstructing this adapter with the same registry and prefix reuses its collectors and diff --git a/formal/guides/PORTING.md b/formal/guides/PORTING.md index 2c23f34a..6156f65f 100644 --- a/formal/guides/PORTING.md +++ b/formal/guides/PORTING.md @@ -240,7 +240,8 @@ Every event has `event`; additional label fields must be JSON primitives. The | `size`, `storedSize` | cache labels + `bytes` | | `compression` | cache labels + `outcome` | | `serialization` | cache labels + `operation`, `seconds` | -| `mismatchWarning` | `cacheNamespace`, `useCase`, `keyType`, `outcome`; optional `cacheKey`, `cachedValueJson`, `sourceValueJson` (string or null) | +| `mismatchWarning` | `cacheNamespace`, `useCase`, `keyType`, `outcome`; optional, each recorded exactly when the warning carries it: `cacheKey`, `cachedValueJson`, `sourceValueJson` and `diffJson` (string or null), `cachedValueAgeSeconds` (number or null; every shadow-profile warning follows a verdict that sampled an age, so that profile's projection requires a number), `mismatchPathsJson` (the `mismatchPaths` array as one JSON string, or null), `mismatchPathCount` (integer or null), `loggingOutcome` (`hook_error` or `render_error`) | +| `shadowMismatchLogging` | `cacheNamespace`, `useCase`, `keyType` (no layer), `outcome` (`logged`, `hook_error`, `render_error` or `logger_error`) | | `readContext` | `index`, `timeoutMs`, `aborted` | | `readAbort`, `writeDispatch` | `index` | | `marker` | `cutoffMs`, `ttlMs` (signed; a `-1` cutoff means no watermark) | @@ -250,11 +251,19 @@ Fixture sentinels are defined by `$defs/behaviorFixture`; `core` and number, `null` (no deadline) or `"default"` (the library default); omitted means 10 ms. `readTimeoutMs` is a number or `"default"`; omitted means 50 ms. `recovery` is `allow`, `deny`, `error` or `default`; `comparator` is `equal`, -`unequal` or `error`. `observe` lists event kinds from the table above, and its -presence, even empty, makes `events` required. `localMaxSize` and +`unequal` or `error`; `mismatchDiff` is `error`, a `shadowMismatchDiff` hook that +fails (a throw, a returned error or a panic, as the port's hook fails). `observe` +lists event kinds from the table above, and its presence, even empty, makes `events` +required. `localMaxSize` and `shadowMaxInFlight` are nonnegative integers; `comparisonMs` and `sourceWorkMs` are nonnegative numbers; `tracked`, `remote`, `shadowHook`, `observerFailure`, -`probeSourceScope` and `localFaultInjection` are booleans. The coordinator +`probeSourceScope` and `localFaultInjection` are booleans. The `faults` command's +`logger` makes the logger fail after the driver records what it received, while +metric observers keep working. `resolve` with `unrenderable: true` and no `value` +settles the loader with the driver's value that has no JSON rendering (a throwing +`toJSON`, a failing `MarshalJSON` or `Serialize`, a non-serializable object); the +driver knows it by identity and reports a call that returns it as +`{"unrenderable": true}`. The coordinator validates its own prepare response, fixture included, against this definition, so it cannot hand a port an unknown sentinel. diff --git a/formal/guides/SPEC.md b/formal/guides/SPEC.md index e97316b2..8241d2e6 100644 --- a/formal/guides/SPEC.md +++ b/formal/guides/SPEC.md @@ -150,7 +150,7 @@ does not establish readability. | Serving ramp | A configured valid TTL implies 100 unless a ramp is supplied; finite [0, 100], admission is sample strictly less than ramp; invalid ramp disables only that layer | | Recovery | Off; zero disables inheritance; a positive maximum age must be a safe integer greater than remote TTL and at most 31,536,000 seconds; invalid optional age disables recovery | | Shadow | Off; valid independent ramp, outcome hook, and capacity are required; invalid optional policy preserves ordinary serving | -| Mismatch logging | Off; malformed logging flag disables warnings | +| Mismatch logging | Off: `value` false and `diff` `off`; each layer's `logMismatches` expands to both leaves, a leaf written beside it overriding the expansion, before the leaf-wise merge; an invalid leaf acts off, a malformed `logMismatches` expands to both leaves off, and one configuration error covers every invalid input | | Read deadline | Runtime -> operation -> instance -> 50 ms; positive safe integer at most 2,147,483,647 ms; invalid runtime budget bypasses caching | | Source deadline | Operation budget, otherwise 60,000 ms; explicit unbounded is allowed; finite budget has the same integer domain as read deadlines | | Local capacity / shadow capacity | 10,000 / 1 per instance; zero local capacity preserves eligible sharing | @@ -258,7 +258,16 @@ a served job owns its detached raw source and retains capacity until settlement. C0 retained while fresh is not reclassified after it ages. C1 confirms payload identity even after freshness expiry or wall rollback; mismatch age clamps to zero, while future-frame diagnostics retain their exact layer and offset. -Mismatch warnings require opt-in and a confirmed verdict. +A mismatch warning requires a confirmed verdict and opted-in logging: `value` +true or `diff` above `off`, as the job captured them at admission. It carries +the field groups those leaves name. Each warning attempt reports one +`shadowMismatchLogging` outcome and logging off reports none: `hook_error` when +the `shadowMismatchDiff` hook (run only with `diff` on) or the built-in walk +fails, else `render_error` when a rendered side has no JSON rendering (an absent +side is not one), else `logger_error` when the logger fails, else `logged`. A +failure nulls the fields it feeds and never falls back to logging more. The +built-in walk (W10) reports nothing for identical renderings and one root entry +for scalars that differ or for an absent side. Invalidation (C32–C39/W09) groups all tracked use-case/argument variants of the same namespace/type/id. It validates arguments before any mutation, advances diff --git a/formal/guides/VALIDATION.md b/formal/guides/VALIDATION.md index 525ea9c1..56995bde 100644 --- a/formal/guides/VALIDATION.md +++ b/formal/guides/VALIDATION.md @@ -107,7 +107,7 @@ inputs before merge, at the tier those inputs need. The manual workflow, one. `replay` generates the corpus, completes every port replay and runs the symbolic lane, about 15 minutes. `mutations` adds the TypeScript, Go and Rust mutation campaigns, about 50 minutes. `full`, the default, adds `check-models` with the -model-fault campaign, about two and a half hours. Port, replay-driver and +model-fault campaign, about three hours. Port, replay-driver and regression-only changes need `replay`, or `mutations` when they edit a file a catalogued mutant rewrites. A change to a model's transitions, invariants or challenges, the kernel library or the Quint toolchain needs `full`, because @@ -239,8 +239,9 @@ its timeout, so catalog growth fails the pull request until the matrix grows. The Go lane bounded the whole run when it had 13 mutants on three shards (25 minutes on a fast runner, 47 to 48 minutes on the slow class, runs 34669546872 and 34670045249; the TypeScript lane -took 17), and its cost grows with the catalog: about 1.6 minutes per Go mutant -and one per TypeScript mutant on a fast runner, twice that on the slow class. +took 17), and its cost grows with the catalog and the corpus: about one minute +per TypeScript mutant on a fast runner and twice that on the slow class, and +about 2.5 and 4.5 minutes per Go mutant (run 40146831443). Each shard measures every unmodified baseline itself, so its evidence stands on the environment it ran in, then measures a contiguous slice of the catalog. A merge job per language reads the shard reports and writes the complete report. It diff --git a/formal/guides/course/01-why-formal.md b/formal/guides/course/01-why-formal.md index 2f89f43e..e3d281b6 100644 --- a/formal/guides/course/01-why-formal.md +++ b/formal/guides/course/01-why-formal.md @@ -76,15 +76,15 @@ Six stages turn a rule into evidence. Every later lesson sits in one of them: ```text "models": 32, - "libraries": 34, + "libraries": 35, "profiles": 17, - "invariants": 253, - "regressions": 468, + "invariants": 258, + "regressions": 489, ``` Read it as: `"models": 32` Quint models, `"profiles": 17` of them [profiles](../GLOSSARY.md#profile-conformance-profile) that ports replay, - `"invariants": 253` checked invariants and `"regressions": 468` named runs. + `"invariants": 258` checked invariants and `"regressions": 489` named runs. 2. Ask where C23 lives: diff --git a/formal/guides/course/07-replay-boundary.md b/formal/guides/course/07-replay-boundary.md index 56e8c6d5..77441344 100644 --- a/formal/guides/course/07-replay-boundary.md +++ b/formal/guides/course/07-replay-boundary.md @@ -64,8 +64,8 @@ action in front. receipt, comparison, next commands); line 141 says it again (`Predictions never choose commands`). - **[protocol.schema.json](../../replay/protocol.schema.json)**: the `$defs` for - `request`, `command`, `behaviorObservation` (line 1780), `settlementReceipt` - (line 2090) and `behaviorFixture` (line 2165). The schema file, not the + `request`, `command`, `behaviorObservation` (line 1894), `settlementReceipt` + (line 2204) and `behaviorFixture` (line 2279). The schema file, not the version number, is the compatibility unit. - **[../PORTING.md](../PORTING.md#shared-coordinator-protocol)**, "Shared coordinator protocol" and "[Observation diff --git a/formal/guides/course/08-typescript-driver.md b/formal/guides/course/08-typescript-driver.md index c34f35b5..0d0a29d3 100644 --- a/formal/guides/course/08-typescript-driver.md +++ b/formal/guides/course/08-typescript-driver.md @@ -59,27 +59,27 @@ Choice `0` of `beginCall` becomes `{ op: "begin" }`; choice `3` of - **[behavior-driver.ts](../../../typescript/test/formal/behavior-driver.ts)**, the driver: - - [lines 54 to 69](../../../typescript/test/formal/behavior-driver.ts): the + - [lines 55 to 70](../../../typescript/test/formal/behavior-driver.ts): the `Input` union, every command a driver understands. - - [lines 73 to 90](../../../typescript/test/formal/behavior-driver.ts): the + - [lines 74 to 91](../../../typescript/test/formal/behavior-driver.ts): the `Observation` record. - - [lines 111 to 112](../../../typescript/test/formal/behavior-driver.ts): the + - [lines 117 to 118](../../../typescript/test/formal/behavior-driver.ts): the rule `no expected observation is passed to this class`. - - [lines 294 to 343](../../../typescript/test/formal/behavior-driver.ts): + - [lines 301 to 354](../../../typescript/test/formal/behavior-driver.ts): `apply` for `begin`; - [lines 345 to 360](../../../typescript/test/formal/behavior-driver.ts): + [lines 356 to 371](../../../typescript/test/formal/behavior-driver.ts): `resolve`, `reject` and `advance`; - [lines 405 to 412](../../../typescript/test/formal/behavior-driver.ts): + [lines 416 to 423](../../../typescript/test/formal/behavior-driver.ts): `release`. - - [lines 440 to 472](../../../typescript/test/formal/behavior-driver.ts): the + - [lines 451 to 483](../../../typescript/test/formal/behavior-driver.ts): the `settle` drain and the receipt; - [lines 481 to 483](../../../typescript/test/formal/behavior-driver.ts): + [lines 492 to 494](../../../typescript/test/formal/behavior-driver.ts): `snapshot()` and `receipt()`. - **[formal-features.test.ts](../../../typescript/test/formal-features.test.ts)**, lines 16 to 53: the `replay` loop. Lines 58 to 60: the `DIALCACHE_FEATURE_TRACE_FILE` and `DIALCACHE_FEATURE_PROFILE` selectors. -- **[features.mjs](../../replay/features.mjs)**: `featureInput` (line 510) and - `assertFeatureObservation` (line 512), shared by every port through the +- **[features.mjs](../../replay/features.mjs)**: `featureInput` (line 617) and + `assertFeatureObservation` (line 619), shared by every port through the coordinator. - **[profiles/source-budgets.mjs](../../replay/profiles/source-budgets.mjs)**: the fixture by init choice, the setup command, and the action decodings. diff --git a/formal/guides/course/09-go-rust-python-drivers.md b/formal/guides/course/09-go-rust-python-drivers.md index 9bd317d8..1f05343c 100644 --- a/formal/guides/course/09-go-rust-python-drivers.md +++ b/formal/guides/course/09-go-rust-python-drivers.md @@ -15,9 +15,9 @@ protocol over its stdin and stdout. | Duty | Go | Rust | Python | | --- | --- | --- | --- | -| Apply a command | `behaviorDriver`'s `apply` at [line 671](../../../go/behavior_driver_test.go) | `Driver`'s `apply` at [line 497](../../../rust/tests/formal/driver.rs), `begin` at [line 759](../../../rust/tests/formal/driver.rs) | `BehaviorDriver`'s `apply` at [line 405](../../../python/tests/formal/driver.py), `begin` at [line 327](../../../python/tests/formal/driver.py) | -| Settle | `settle` at [line 543](../../../go/behavior_driver_test.go) | `settle` at [line 711](../../../rust/tests/formal/driver.rs) | `settle` at [line 522](../../../python/tests/formal/driver.py) | -| Report | `observation` at [line 519](../../../go/behavior_driver_test.go), `receipt` at [line 528](../../../go/behavior_driver_test.go) | `observation` at [line 1002](../../../rust/tests/formal/driver.rs), `receipt` at [line 1006](../../../rust/tests/formal/driver.rs) | `observe` at [line 531](../../../python/tests/formal/driver.py), `receipt` at [line 534](../../../python/tests/formal/driver.py) | +| Apply a command | `behaviorDriver`'s `apply` at [line 671](../../../go/behavior_driver_test.go) | `Driver`'s `apply` at [line 521](../../../rust/tests/formal/driver.rs), `begin` at [line 787](../../../rust/tests/formal/driver.rs) | `BehaviorDriver`'s `apply` at [line 453](../../../python/tests/formal/driver.py), `begin` at [line 370](../../../python/tests/formal/driver.py) | +| Settle | `settle` at [line 543](../../../go/behavior_driver_test.go) | `settle` at [line 739](../../../rust/tests/formal/driver.rs) | `settle` at [line 571](../../../python/tests/formal/driver.py) | +| Report | `observation` at [line 519](../../../go/behavior_driver_test.go), `receipt` at [line 528](../../../go/behavior_driver_test.go) | `observation` at [line 1036](../../../rust/tests/formal/driver.rs), `receipt` at [line 1040](../../../rust/tests/formal/driver.rs) | `observe` at [line 580](../../../python/tests/formal/driver.py), `receipt` at [line 583](../../../python/tests/formal/driver.py) | | Transport | `replayCoordinator` in [replay_coordinator_test.go](../../../go/replay_coordinator_test.go): `prepare` at [line 183](../../../go/replay_coordinator_test.go), `replay` at [line 234](../../../go/replay_coordinator_test.go) | `Coordinator` in [transport.rs](../../../rust/tests/formal/transport.rs): `spawn` at [line 96](../../../rust/tests/formal/transport.rs), `call` at [line 187](../../../rust/tests/formal/transport.rs), `prepare` at [line 253](../../../rust/tests/formal/transport.rs) | `Coordinator` in [coordinator.py](../../../python/tests/formal/coordinator.py): `request` at [line 31](../../../python/tests/formal/coordinator.py), `replay` at [line 53](../../../python/tests/formal/coordinator.py) | The transports share three habits. Request ids strictly increase for the life diff --git a/formal/guides/course/13-catalogs-and-ledgers.md b/formal/guides/course/13-catalogs-and-ledgers.md index 61e7f330..463dd5ed 100644 --- a/formal/guides/course/13-catalogs-and-ledgers.md +++ b/formal/guides/course/13-catalogs-and-ledgers.md @@ -97,8 +97,8 @@ You will edit a tracked guide and undo it at the end. ```text { sources: 50, - tests: 564, - sections: 215, + tests: 633, + sections: 216, reviewedGuides: 19, guideSections: 195 } diff --git a/formal/guides/course/14-lanes-and-local-runs.md b/formal/guides/course/14-lanes-and-local-runs.md index a7656143..79cacdca 100644 --- a/formal/guides/course/14-lanes-and-local-runs.md +++ b/formal/guides/course/14-lanes-and-local-runs.md @@ -26,7 +26,7 @@ four shards. These are gates on a change, not a completion claim. for any branch. Its jobs are the six stages of lesson 1 as separate lanes: - `check-models` (`make formal-check`): typecheck and run every model, every - regression and every challenge; budget 150 minutes. + regression and every challenge; budget 210 minutes. - `generate` (`make formal-generate`): the corpus and the witness evidence. - One parity lane per port: `make formal-ts`, `formal-go`, `formal-rust` and `formal-python`. @@ -70,9 +70,9 @@ command, so the step name tells you which lesson's check fired. the `mode` input that selects the lanes, and the `timeout-minutes` budget on each. - **[validation.mjs](../../tools/validation.mjs)**, lines 11 to 12: `check` and - `formal` as lists of other targets. Line 195: the five checks of `audit`. - Lines 197 to 201: its fixture, style (`check-style.mjs`), course, read-view - and scope steps. Line 202: `smoke`. Line 212: `formal-check`. + `formal` as lists of other targets. Line 200: the five checks of `audit`. + Lines 202 to 206: its fixture, style (`check-style.mjs`), course, read-view + and scope steps. Line 207: `smoke`. Line 217: `formal-check`. - **[../VALIDATION.md](../VALIDATION.md#choosing-a-run)**, "Choosing a run": the task-to-command table, and the rest of the page for reading a completion report and a mutation report. diff --git a/formal/guides/course/16-capstone-run.md b/formal/guides/course/16-capstone-run.md index 08c16c60..3113b4bf 100644 --- a/formal/guides/course/16-capstone-run.md +++ b/formal/guides/course/16-capstone-run.md @@ -59,7 +59,7 @@ point is that the same integer means the same command in every port. lines 256 to 260: `abandonedSourceCannotReplaceSuccessfulRetryTest`, the nearest existing run; compare its shape with the new one. - **[conformance-observations.qnt](../../models/conformance-observations.qnt)**, - lines 8 to 17: the caller outcome codes, `CALL_PENDING` first. + lines 8 to 19: the caller outcome codes, `CALL_PENDING` first. - **[deadlines.qnt](../../models/kernel/deadlines.qnt)**, lines 181 to 197: `releaseLocal` stamps the retry's deadline from `state.now`; `advanceLocal` delivers it. diff --git a/formal/guides/course/17-capstone-ripple.md b/formal/guides/course/17-capstone-ripple.md index 121cf377..4f4279a3 100644 --- a/formal/guides/course/17-capstone-ripple.md +++ b/formal/guides/course/17-capstone-ripple.md @@ -45,7 +45,7 @@ stronger, that replayed it in all four ports. - **[generated-fixtures.mjs](../../tools/generated-fixtures.mjs)**: the `--verify` mode the audit lane runs - ([validation.mjs](../../tools/validation.mjs) line 197, `--verify`) and the + ([validation.mjs](../../tools/validation.mjs) line 202, `--verify`) and the `--write` path behind `generate-artifacts.mjs`. - **[../../generated/README.md](../../generated/README.md)**: the two regeneration commands and the rule "never edit a file in this directory". diff --git a/formal/guides/course/README.md b/formal/guides/course/README.md index 76b94d4a..e619e8b6 100644 --- a/formal/guides/course/README.md +++ b/formal/guides/course/README.md @@ -147,7 +147,7 @@ so the drift is caught by the check, not by a learner: symbol is no longer on those lines. Fix the citation, then reread the lesson: the code may have moved for a reason the lesson should mention. - **A quoted count comes from a check.** Quote a number only as `node - formal/tools/execution.mjs` prints it (`"regressions": 468`) or as `node + formal/tools/execution.mjs` prints it (`"regressions": 489`) or as `node formal/tools/check-source-audit.mjs` prints it (`guideSections: 195`); the check compares both with today's output. When an exercise changes a count, say so in words rather than quoting the new number. diff --git a/formal/models/conformance-observations.qnt b/formal/models/conformance-observations.qnt index bae58dd2..299b1051 100644 --- a/formal/models/conformance-observations.qnt +++ b/formal/models/conformance-observations.qnt @@ -15,6 +15,8 @@ module conformance_observations { pure val VALUE_FALSE = 7 pure val VALUE_ZERO = 8 pure val VALUE_EMPTY_STRING = 9 + // A source result with no JSON rendering (D13; the shadow profile's). + pure val VALUE_UNRENDERABLE = 12 pure val values = List( VALUE_ONE, VALUE_TWO, VALUE_ABSENT, VALUE_NULL, diff --git a/formal/models/dialcache-admission-conformance.qnt b/formal/models/dialcache-admission-conformance.qnt index 06d1fb6a..396f1ad0 100644 --- a/formal/models/dialcache-admission-conformance.qnt +++ b/formal/models/dialcache-admission-conformance.qnt @@ -64,7 +64,10 @@ module dialcache_admission_conformance { // The instance's shadow policy as read when a hit is judged: the outcome // hook installed, two slots per instance, a 10 ms job budget, no mismatch // logging; the cohort selection consulted is the one captured for the flight. - pure val SHADOWING: Shadow::Shadowing = { hook: true, selected: true, capacity: JOB_CAPACITY, budgetMs: SHADOW_TIMEOUT_MS, logging: false } + pure val SHADOWING: Shadow::Shadowing = { + hook: true, selected: true, capacity: JOB_CAPACITY, budgetMs: SHADOW_TIMEOUT_MS, + logging: Shadow::LOGGING_OFF, diffHook: false + } // State: the held remote shape with the served job registry, its budgets // and the cohort selections captured for pending flights, and the policy diff --git a/formal/models/dialcache-dark-layers-conformance.qnt b/formal/models/dialcache-dark-layers-conformance.qnt index 310a4a4e..320b2e0d 100644 --- a/formal/models/dialcache-dark-layers-conformance.qnt +++ b/formal/models/dialcache-dark-layers-conformance.qnt @@ -65,7 +65,11 @@ module dialcache_dark_layers_conformance { keysPerInstance: KEYS, keysPerScope: KEYS, persistentContexts: SCOPES, operationsPerEntity: 1, probeSourceScope: true, policyProvider: true } - type Channel = { inspections: List[Flights::Inspection], configErrors: int, warnings: int, ages: List[int], coalesced: List[str], fallbackErrors: List[str], futureOffsets: List[Diagnostics::FutureOffset] } + type Channel = { + inspections: List[Flights::Inspection], configErrors: int, warnings: int, ages: List[int], + coalesced: List[str], fallbackErrors: List[str], futureOffsets: List[Diagnostics::FutureOffset], + mismatchWarnings: List[Shadow::MismatchWarning], loggingOutcomes: List[str] + } type State = { o: Observation, d: Channel, io: RemoteIO::IO, now: int, skew: int, remoteAvailable: bool, tracked: bool, capacity: int, @@ -107,7 +111,10 @@ module dialcache_dark_layers_conformance { } pure def selectedFor(choice: int): bool = choice != DARK_ALL_NO_SHADOW pure def shadowing(selected: bool): Shadow::Shadowing = - { hook: true, selected: selected, capacity: JOB_CAPACITY, budgetMs: SOURCE_BUDGET_MS, logging: false } + { + hook: true, selected: selected, capacity: JOB_CAPACITY, budgetMs: SOURCE_BUDGET_MS, + logging: Shadow::LOGGING_OFF, diffHook: false + } pure def instance(context: int): int = if (context == 2 or context == 4) 1 else 0 pure def call(choice: int): Call = { instance: instance(choice / KEYS), key: choice % KEYS, context: choice / KEYS, enabled: true, keyFailed: false } @@ -127,7 +134,10 @@ module dialcache_dark_layers_conformance { pure def invalidationBuffer(choice: int): int = (choice % 2) * FUTURE_BUFFER_MS pure val INITIAL: State = { - o: empty, d: { inspections: List(), configErrors: 0, warnings: 0, ages: List(), coalesced: List(), fallbackErrors: List(), futureOffsets: List() }, + o: empty, d: { + inspections: List(), configErrors: 0, warnings: 0, ages: List(), coalesced: List(), fallbackErrors: List(), + futureOffsets: List(), mismatchWarnings: List(), loggingOutcomes: List() + }, io: RemoteIO::NO_IO, now: 0, skew: WALL_ORIGIN_MS, remoteAvailable: true, tracked: true, capacity: KEYS, readFailed: false, dumpFailed: false, writeFailed: false, loadFailed: false, @@ -364,8 +374,10 @@ module dialcache_dark_layers_conformance { // Joint (clock x shadow read): a future offset is reported only after a wall rollback. val futureOffsetsFollowRollbacks = s.d.futureOffsets.length() == 0 or s.skew < WALL_ORIGIN_MS // The profile's own restrictions, executable: no mismatch logging and no - // malformed policy, so the channel's warnings and configuration errors stay zero. + // malformed policy, so the channel's warnings, their content and logging + // outcomes, and its configuration errors stay empty. val channelRestrictionsHold = s.d.warnings == 0 and s.d.configErrors == 0 + and s.d.mismatchWarnings.length() == 0 and s.d.loggingOutcomes.length() == 0 val agesRequireVerdicts = s.d.ages.length() == s.o.shadow.select(label => label == "match" or label == "mismatch").length() // Every reply the decoder produces is one the library can be configured with (C10). // One knob (C6): the job budget is the source budget, as the library derives it from the call's fallback timeout. diff --git a/formal/models/dialcache-shadow-conformance.qnt b/formal/models/dialcache-shadow-conformance.qnt index 9983e6e7..699205ea 100644 --- a/formal/models/dialcache-shadow-conformance.qnt +++ b/formal/models/dialcache-shadow-conformance.qnt @@ -7,6 +7,7 @@ module dialcache_shadow_conformance { import serving as Serving from "./kernel/serving" import remote_io as RemoteIO from "./kernel/remote-io" import remote_writes as Writes from "./kernel/remote-writes" + import mismatch_logging.* from "./kernel/mismatch-logging" import shadow as Shadow from "./kernel/shadow" import deadlines as Deadlines from "./kernel/deadlines" import diagnostics as Diagnostics from "./kernel/diagnostics" @@ -18,7 +19,8 @@ module dialcache_shadow_conformance { // Model boundary ----------------------------------------------------------- // One ramped-down caller source and its selected dark C0/source/C1 job. // Selected rules from C47/C49-C54 govern admission, comparison, fill authority, - // and timeout ownership; C57/C60 cover snapshot age and captured logging. + // and timeout ownership; C57/C60 cover snapshot age, the captured logging + // leaves and each confirmed-mismatch warning's content and logging outcome. // See formal/generated/PROFILES.md (shadow) for fixtures and payloads and BEHAVIOR.md "Generated profiles". // Each source/read/decode/serialize/write can remain externally pending. This // profile excludes serving hits, request/local storage, and overlapping jobs. @@ -34,11 +36,13 @@ module dialcache_shadow_conformance { // caller's source deadline and the layer its failure is attributed to), the // held fill (remote_writes), the frames a seed stores and the watermark an // invalidation raises with its future buffer (remote_frames, serving), the - // wall skew (clock). This module owns the bounded state, the fixture + // wall skew (clock), and the logging inputs resolved to the leaves a job + // captures (mismatch_logging). This module owns the bounded state, the fixture // decoding (the comparator, its elapsed time and the source work an init - // choice names; the hook and the default logging; the payload code a seed - // choice names; the policy facts a runtime overlay sets), the input choices - // and the guards. + // choice names; the hooks and the default logging inputs; the payload code a + // seed choice names; the policy facts and logging inputs a runtime overlay + // sets), the input choices, the guards and the verdict receipts its + // warning properties read. // Trace encodings and fixture bounds --------------------------------------- pure val COMPARATOR_COUNT = 4 @@ -48,12 +52,25 @@ module dialcache_shadow_conformance { pure val FIXTURE_SLOW_ERROR = 10 pure val FIXTURE_SOURCE_WORK_BEFORE_DEADLINE = 11 pure val FIXTURE_SOURCE_WORK_AT_DEADLINE = 12 + pure val FIXTURE_DIFF_HOOK = 13 + pure val FIXTURE_DEFAULT_PATHS = 14 + pure val FIXTURE_DEFAULT_VALUE_FALSE = 15 pure val SHADOW_ENABLED = 0 pure val SHADOW_DISABLED = 1 pure val SHADOW_INVALID = 2 pure val LOGGING_DISABLED = 0 pure val LOGGING_ENABLED = 1 pure val LOGGING_INVALID = 2 + pure val LOGGING_PATHS = 3 + pure val LOGGING_VALUES = 4 + pure val LOGGING_VALUE_ALONE = 5 + pure val LOGGING_TRUE_WITHOUT_VALUE = 6 + pure val LOGGING_FALSE_WITH_PATHS = 7 + pure val LOGGING_INVALID_DIFF = 8 + pure val LOGGING_INVALID_VALUE = 9 + pure val LOGGING_INVALID_SHORTHAND_WITH_VALUES = 10 + pure val LOGGING_DIFF_OFF = 11 + pure val LOGGING_VALUE_FALSE = 12 pure val JOB_BUDGET_MS = 10 pure val SOURCE_BUDGET_MS = 10 pure val FRESHNESS_MS = 60000 @@ -91,20 +108,41 @@ module dialcache_shadow_conformance { pure val TEXT_UNICODE = Payloads::UNICODE + DECODED_UNICODE pure val BINARY_UNICODE = Payloads::BINARY + Payloads::UNICODE + DECODED_UNICODE - // Shadow profile v3 retains explicit external inputs and future timestamp - // samples as well as verdict age, captured logging, and configuration errors. + // Shadow profile v4 retains explicit external inputs and future timestamp + // samples as well as verdict age and configuration errors, and adds each + // confirmed-mismatch warning's content and its logging outcome. type ShadowDiagnostics = { configErrors: int, warnings: int, ages: List[int], coalesced: List[str], fallbackErrors: List[str], - futureOffsets: List[Diagnostics::FutureOffset] - } + futureOffsets: List[Diagnostics::FutureOffset], + mismatchWarnings: List[Shadow::MismatchWarning], loggingOutcomes: List[str] + } + // The logging inputs as written, one config layer each (`shadow` in the + // fixture's policy or in a runtime overlay): the `logMismatches` shorthand, + // then the `mismatchLogging` leaves `value` and `diff`. "invalid" is a value + // of the wrong type for any of them; "" leaves it unwritten. + pure def loggingInput(shorthand: str, value: str, diff: str): LoggingInput = + { shorthand: shorthand, value: value, diff: diff } + pure val LOG_MISMATCHES = loggingInput("true", "", "") + // The runtime logging overlays by `logPolicy` choice; choices 6 and 7 are + // the first two rows of the record's shorthand table, and choices 11 and 12 + // write one leaf off. + pure val LOGGING_OVERLAYS = List( + loggingInput("false", "", ""), loggingInput("true", "", ""), loggingInput("invalid", "", ""), + loggingInput("", "", "paths"), loggingInput("", "", "values"), loggingInput("false", "true", ""), + loggingInput("true", "false", ""), loggingInput("false", "", "paths"), + loggingInput("", "true", "invalid"), loggingInput("", "invalid", "paths"), loggingInput("invalid", "", "values"), + loggingInput("", "", "off"), loggingInput("", "false", "") + ) // State: the layered shape with its held remote effects, deadlines, held // fills and jobs, the two driver channels (o, d), the io channel the held // shape carries, and the fixture's facts: the comparator, its elapsed time - // and the source work the init choice names, the hook and the default - // logging, and the shadow policy in force (the ramp selecting, the ramp - // malformed, logging on, logging malformed), decoded once at each overlay. + // and the source work the init choice names, the outcome and diff hooks and + // the default logging inputs, and the shadow policy in force (the ramp + // selecting, the ramp malformed, decoded once at each overlay, and the + // overlay's logging inputs, resolved at admission), the logger's failure + // input and the receipt of each verdict (`shadow::recordVerdictInputs`). type State = { o: Observation, d: ShadowDiagnostics, io: RemoteIO::IO, now: int, skew: int, remoteAvailable: bool, tracked: bool, capacity: int, @@ -116,8 +154,9 @@ module dialcache_shadow_conformance { reads: List[RemoteIO::HeldRead], loads: List[RemoteIO::HeldLoad], loaders: List[int], retained: List[Recovery::Snapshot], sourceBudget: int, deadlines: List[Deadlines::Due], drained: List[bool], sourceLayers: List[str], dumps: List[Writes::HeldDump], writes: List[Writes::HeldWrite], jobs: List[Shadow::Job], - comparator: int, comparisonMs: int, sourceWorkMs: int, hook: bool, defaultLog: bool, - selected: bool, rampInvalid: bool, logging: bool, loggingInvalid: bool + comparator: int, comparisonMs: int, sourceWorkMs: int, hook: bool, diffHook: bool, + defaultLogging: LoggingInput, selected: bool, rampInvalid: bool, overlayLogging: LoggingInput, + loggerFailed: bool, verdicts: List[Shadow::Verdict] } var s: State var input: { name: str, choice: int } @@ -127,7 +166,10 @@ module dialcache_shadow_conformance { // with mismatch logging enabled; fixture 8 omits the outcome hook. // Fixtures 9/10 let successful/failing comparison consume the job budget. // Fixtures 11/12 consume 9/10 ms of synchronous external source work before - // deferred job dispatch. + // deferred job dispatch. Fixture 13 installs a throwing `shadowMismatchDiff` + // hook with mismatch logging enabled; fixtures 14/15 write the default + // leaves `diff: "paths"` and `value: false` (the shorthand table's last two + // rows). 13 to 15 use the library comparator. pure def comparatorOf(choice: int): int = if (choice >= FIXTURE_SOURCE_WORK_BEFORE_DEADLINE) Shadow::COMPARE_VALUES else if (choice == FIXTURE_SLOW_ERROR) Shadow::COMPARE_ERROR else choice % COMPARATOR_COUNT @@ -136,7 +178,12 @@ module dialcache_shadow_conformance { if (choice == FIXTURE_SOURCE_WORK_AT_DEADLINE) JOB_BUDGET_MS else if (choice == FIXTURE_SOURCE_WORK_BEFORE_DEADLINE) JOB_BUDGET_MS - 1 else 0 pure def hookOf(choice: int): bool = choice != FIXTURE_NO_HOOK - pure def defaultLogOf(choice: int): bool = choice >= FIRST_LOGGING_FIXTURE and choice < FIXTURE_NO_HOOK + pure def diffHookOf(choice: int): bool = choice == FIXTURE_DIFF_HOOK + pure def defaultLoggingOf(choice: int): LoggingInput = + if ((choice >= FIRST_LOGGING_FIXTURE and choice < FIXTURE_NO_HOOK) or choice == FIXTURE_DIFF_HOOK) LOG_MISMATCHES + else if (choice == FIXTURE_DEFAULT_PATHS) loggingInput("", "", "paths") + else if (choice == FIXTURE_DEFAULT_VALUE_FALSE) loggingInput("", "false", "") + else NO_LOGGING_INPUT // The payload a seed choice stores, and the choice naming the other form of // a stored payload (the drivers' `reencode`). pure def seedPayload(choice: int): int = @@ -147,7 +194,10 @@ module dialcache_shadow_conformance { // Initial fixture: no frame, no watermark, empty storage, the ramp selecting, // logging as the fixture defaults it. Policy changes are captured at admission. pure def initialState(choice: int): State = { - o: empty, d: { configErrors: 0, warnings: 0, ages: List(), coalesced: List(), fallbackErrors: List(), futureOffsets: List() }, + o: empty, d: { + configErrors: 0, warnings: 0, ages: List(), coalesced: List(), fallbackErrors: List(), futureOffsets: List(), + mismatchWarnings: List(), loggingOutcomes: List() + }, io: RemoteIO::NO_IO, now: INITIAL_NOW_MS, skew: WALL_SKEW_MS, remoteAvailable: true, tracked: true, capacity: 0, readFailed: false, dumpFailed: false, writeFailed: false, loadFailed: false, @@ -159,15 +209,15 @@ module dialcache_shadow_conformance { sourceBudget: SOURCE_BUDGET_MS, deadlines: List(), drained: List(), sourceLayers: List(), dumps: List(), writes: List(), jobs: List(), comparator: comparatorOf(choice), comparisonMs: comparisonMsOf(choice), sourceWorkMs: sourceWorkOf(choice), - hook: hookOf(choice), defaultLog: defaultLogOf(choice), - selected: true, rampInvalid: false, logging: defaultLogOf(choice), loggingInvalid: false + hook: hookOf(choice), diffHook: diffHookOf(choice), defaultLogging: defaultLoggingOf(choice), + selected: true, rampInvalid: false, overlayLogging: NO_LOGGING_INPUT, loggerFailed: false, verdicts: List() } action initialize(choice: int): bool = all { input' = { name: "init", choice: choice }, s' = initialState(choice) } action init = { - nondet choice = 0.to(FIXTURE_SOURCE_WORK_AT_DEADLINE).oneOf() + nondet choice = 0.to(FIXTURE_DEFAULT_VALUE_FALSE).oneOf() initialize(choice) } @@ -177,22 +227,37 @@ module dialcache_shadow_conformance { // late result drains); the drivers settle the latest loader and release the // latest held read, decode, dump or write. val sourcePending = s.loaders.length() > 0 and Deadlines::pending(s, RemoteIO::latestOrdinal(s)) + // A source result with no JSON rendering is offered only while no live job + // could still fill with it: every job has timed out or acquired its C0. A + // dark fill of it would fail natively at its dump, which the model does not + // state, so the environment never offers that history. + val unrenderableCannotFill = s.jobs.indices().forall(i => s.jobs.nth(i).timedOut or s.jobs.nth(i).cached != NO_VALUE) // Caller source and job admission ------------------------------------------ action beginCall = all { input' = { name: "beginCall", choice: -1 }, s.jobs.length() == 0, not(sourcePending), s.o.calls.length() < MAX_CALLS, s' = Shadow::beginDark(s, LAYOUT, CALL, DARK, TTLS, Deadlines::UNBOUNDED, - { hook: s.hook, selected: s.selected, capacity: 1, budgetMs: JOB_BUDGET_MS, logging: s.logging }, - { ramp: s.rampInvalid, logging: s.loggingInvalid }, s.sourceWorkMs) + { + hook: s.hook, selected: s.selected, capacity: 1, budgetMs: JOB_BUDGET_MS, + logging: resolveLogging(mergeLogging(s.defaultLogging, s.overlayLogging)).logging, + diffHook: s.diffHook + }, + { + ramp: s.rampInvalid, + logging: resolveLogging(mergeLogging(s.defaultLogging, s.overlayLogging)).invalid + }, + s.sourceWorkMs) } action resolveAt(choice: int): bool = all { input' = { name: "resolveLoader", choice: choice }, - sourcePending, + sourcePending, choice != VALUE_UNRENDERABLE or unrenderableCannotFill, s' = Shadow::settleHeld(s, LAYOUT, RemoteIO::latestOrdinal(s), choice, false) } action resolveLoader = { - nondet choice = Set(VALUE_ONE, VALUE_TWO).oneOf() + nondet choice = ( + if (unrenderableCannotFill) Set(VALUE_ONE, VALUE_TWO, VALUE_UNRENDERABLE) else Set(VALUE_ONE, VALUE_TWO) + ).oneOf() resolveAt(choice) } action rejectLoader = all { @@ -205,12 +270,17 @@ module dialcache_shadow_conformance { action releaseRead = all { input' = { name: "releaseRead", choice: -1 }, Shadow::latestRead(s) >= 0, - s' = Shadow::releaseRead(s, LAYOUT, Shadow::latestRead(s), s.readFailed) + s' = Shadow::recordVerdictInputs(s, + Shadow::releaseReadWithLogger(s, LAYOUT, Shadow::latestRead(s), s.readFailed, s.loggerFailed), + Shadow::readerOf(s, Shadow::latestRead(s)), s.loggerFailed) } action releaseLoad = all { input' = { name: "releaseLoad", choice: -1 }, Shadow::latestLoad(s) >= 0, - s' = Shadow::releaseLoad(s, LAYOUT, Shadow::latestLoad(s), s.loadFailed, { comparator: s.comparator, elapsedMs: s.comparisonMs }, Deadlines::UNBOUNDED) + s' = Shadow::recordVerdictInputs(s, + Shadow::releaseLoad(s, LAYOUT, Shadow::latestLoad(s), s.loadFailed, + { comparator: s.comparator, elapsedMs: s.comparisonMs }, Deadlines::UNBOUNDED), + Shadow::decoderOf(s, Shadow::latestLoad(s)), s.loggerFailed) } action releaseDump = all { input' = { name: "releaseDump", choice: -1 }, @@ -285,18 +355,22 @@ module dialcache_shadow_conformance { advanceWallBy(choice) } // A runtime overlay replaces the previous one: a logging overlay leaves the - // ramp at the fixture's 100, a ramp overlay leaves logging at the fixture's default. + // ramp at the fixture's 100, a ramp overlay leaves logging to the fixture's + // defaults. The logging inputs are kept as written and resolved at admission. action setLogging(choice: int): bool = all { input' = { name: "logPolicy", choice: choice }, - s' = { logging: choice == LOGGING_ENABLED, loggingInvalid: choice == LOGGING_INVALID, selected: true, rampInvalid: false, ...s } + s' = { overlayLogging: LOGGING_OVERLAYS.nth(choice), selected: true, rampInvalid: false, ...s } } action logPolicy = { - nondet choice = Set(LOGGING_DISABLED, LOGGING_ENABLED, LOGGING_INVALID).oneOf() + nondet choice = 0.to(LOGGING_VALUE_FALSE).oneOf() setLogging(choice) } action shadowPolicyAt(choice: int): bool = all { input' = { name: "shadowPolicy", choice: choice }, - s' = { selected: choice == SHADOW_ENABLED, rampInvalid: choice == SHADOW_INVALID, logging: s.defaultLog, loggingInvalid: false, ...s } + s' = { + selected: choice == SHADOW_ENABLED, rampInvalid: choice == SHADOW_INVALID, + overlayLogging: NO_LOGGING_INPUT, ...s + } } action shadowPolicy = { nondet choice = Set(SHADOW_ENABLED, SHADOW_DISABLED, SHADOW_INVALID).oneOf() @@ -334,17 +408,33 @@ module dialcache_shadow_conformance { nondet choice = Set(0, 1).oneOf() writeFaultAt(choice) } + // The application logger fails when it receives a warning; the warning was + // still handed to it (D15: a synchronous throw). + action loggerFaultAt(choice: int): bool = all { + input' = { name: "loggerFault", choice: choice }, + s' = { loggerFailed: choice == 1, ...s } + } + action loggerFault = { + nondet choice = Set(0, 1).oneOf() + loggerFaultAt(choice) + } // Exploration schedule: retain every progress/environment action. // At C1, favor completion and logging changes while retaining every // replacement, failure, clock, and admission-policy transition. Unicode seed // choices add exploration frequency, not another payload representation. + // A third group weights the logging inputs (the overlays and the logger's + // failure), so each leaf combination reaches a confirmed mismatch. action step = if (Shadow::confirming(s)) any { - releaseRead, logPolicy, reencode, advanceWall, + releaseRead, logPolicy, reencode, advanceWall, loggerFault, any { seed, seedUnicode, invalidate, readFault, advance, rollbackWall, shadowPolicy } } else any { any { beginCall, resolveLoader, rejectLoader, releaseRead, releaseLoad, releaseDump, releaseWrite }, - any { advance, seed, seedUnicode, invalidate, readFault, loadFault, dumpFault, writeFault, rollbackWall, logPolicy, shadowPolicy } + any { + advance, seed, seedUnicode, invalidate, readFault, loadFault, dumpFault, writeFault, rollbackWall, + logPolicy, shadowPolicy + }, + any { logPolicy, loggerFault } } // Invariants: independent checks over the public channels and the library records. @@ -356,10 +446,89 @@ module dialcache_shadow_conformance { val agesRequireVerdicts = s.d.ages.length() == s.o.shadow.select(x => x == "match" or x == "mismatch").length() val warningsRequireMismatch = s.d.warnings <= s.o.shadow.select(x => x == "mismatch").length() val expiredSourceWorkSkipsRedis = s.sourceWorkMs < JOB_BUDGET_MS or s.o.reads == 0 - // The environment restriction stated: an invalid logging value never also - // enables logging, so no job captures an authority from a malformed value - // (the former invalidLoggingCannotBeCaptured over the captured pair). - val invalidLoggingCannotBeCaptured = not(s.loggingInvalid and s.logging) + // The logging inputs in force for each leaf, written independently of + // mismatch_logging::mergeLogging: the overlay's own leaf when written, else + // what its shorthand stands for, else the default's leaf, else its + // shorthand's. A `true` shorthand stands for "true" and "values", `false` + // for "false" and "off", and a malformed one for "invalid", the input it + // came from. + pure def standsFor(shorthand: str, valueLeaf: bool): str = + if (shorthand == "") "" else if (shorthand == "true") (if (valueLeaf) "true" else "values") + else if (shorthand == "false") (if (valueLeaf) "false" else "off") else "invalid" + pure def layerLeaf(written: LoggingInput, valueLeaf: bool): str = { + val leaf = if (valueLeaf) written.value else written.diff + if (leaf != "") leaf else standsFor(written.shorthand, valueLeaf) + } + def inForce(valueLeaf: bool): str = { + val overlay = layerLeaf(s.overlayLogging, valueLeaf) + if (overlay != "") overlay else layerLeaf(s.defaultLogging, valueLeaf) + } + // The leaves a job captures at admission (record table and D10): `value` + // exactly when its input in force is "true", `diff` the level its input + // names and off for anything else, and the fixture's diff hook. A leaf + // whose input in force is not a valid value (written with the wrong type, + // or standing for a malformed shorthand) is therefore captured off, + // whatever the other leaf captures (C60). + val capturedLoggingFollowsLayers = input.name != "beginCall" or s.jobs.indices().forall(i => { + val job = s.jobs.nth(i) + val diff = inForce(false) + job.logging == { value: inForce(true) == "true", diff: if (Set("paths", "values").contains(diff)) diff else "off" } + and job.diffHook == s.diffHook + }) + // Over the verdict receipts (`shadow::recordVerdictInputs`): the verdicts + // whose warning is attempted, a confirmed mismatch whose job captured + // either leaf on, by index among the verdicts and so among `d.ages`. + pure def optsIn(verdict: Shadow::Verdict): bool = + verdict.outcome == "mismatch" and (verdict.logging.value or verdict.logging.diff != "off") + def attempts: List[int] = range(0, s.verdicts.length()).select(k => optsIn(s.verdicts.nth(k))) + // One warning per attempt and none otherwise, so none with logging off + // (D11): the receipts are the verdicts the ages sample, and the attempts + // are exactly the warnings. + val noWarningWithLoggingOff = s.verdicts.length() == s.d.ages.length() + and attempts.length() == s.d.mismatchWarnings.length() + // One `shadowMismatchLogging` outcome per warning attempt (D11). + val oneLoggingOutcomePerWarning = s.d.loggingOutcomes.length() == s.d.mismatchWarnings.length() + and s.d.warnings == s.d.mismatchWarnings.length() + // A value field restated: left out with `value` off, null for a side with no JSON rendering. + pure def valueFieldOf(verdict: Shadow::Verdict, code: int): int = + if (not(verdict.logging.value)) Shadow::OMITTED else if (code == VALUE_UNRENDERABLE) Shadow::NO_RENDERING else code + // The diff fields fail closed at a level above off when the hook replaces + // the walk (it throws) or a side has no rendering; else two equal scalars + // differ nowhere and two different ones at the root, both sides carried. + pure def contentAgrees(verdict: Shadow::Verdict, warning: Shadow::MismatchWarning): bool = { + val diffOn = verdict.logging.diff != "off" + val unrenderable = verdict.cached == VALUE_UNRENDERABLE or verdict.result == VALUE_UNRENDERABLE + val failed = diffOn and (verdict.diffHook or unrenderable) + val differs = diffOn and not(failed) and verdict.cached != verdict.result + val root = { path: "", cached: verdict.cached, source: verdict.result } + warning.cachedValue == valueFieldOf(verdict, verdict.cached) + and warning.sourceValue == valueFieldOf(verdict, verdict.result) + and warning.diff == verdict.logging.diff and warning.diffNull == failed + and warning.count == (if (differs) 1 else 0) and warning.paths == (if (differs) List("") else List()) + and warning.entries == (if (differs and verdict.logging.diff == "values") List(root) else List()) + } + // Each warning carries the field groups its job captured and their content + // over its verdict's inputs (D12). + val warningFieldsMatchCapturedLeaves = not(noWarningWithLoggingOff) + or s.d.mismatchWarnings.indices().forall(i => + contentAgrees(s.verdicts.nth(attempts.nth(i)), s.d.mismatchWarnings.nth(i))) + // The outcome restated (D11, D15): `hook_error` when the hook ran (`diff` + // on), over `render_error` for a side with no rendering, over + // `logger_error`, else `logged`; the warning names the first two. + pure def outcomeOf(verdict: Shadow::Verdict): str = + if (verdict.logging.diff != "off" and verdict.diffHook) "hook_error" + else if (verdict.cached == VALUE_UNRENDERABLE or verdict.result == VALUE_UNRENDERABLE) "render_error" + else if (verdict.loggerFailed) "logger_error" else "logged" + val loggingOutcomePrecedence = not(noWarningWithLoggingOff and oneLoggingOutcomePerWarning) + or s.d.mismatchWarnings.indices().forall(i => { + val outcome = outcomeOf(s.verdicts.nth(attempts.nth(i))) + s.d.loggingOutcomes.nth(i) == outcome + and s.d.mismatchWarnings.nth(i).loggingOutcome + == (if (Set("hook_error", "render_error").contains(outcome)) outcome else "") + }) + // A warning's `cachedValueAgeSeconds` is the age its verdict sampled (C57). + val warningAgeIsVerdictAge = not(noWarningWithLoggingOff) + or s.d.mismatchWarnings.indices().forall(i => s.d.mismatchWarnings.nth(i).ageMs == s.d.ages.nth(attempts.nth(i))) // Over the library records, one step from the former text: every held fill // belongs to a job whose C0 found nothing and whose source was accepted (the // former writesRequireMiss over the fill phase). @@ -415,9 +584,10 @@ module dialcache_shadow_conformance { val oneExecution = s.jobs.length() <= 1 and s.sources.select(source => source.result == CALL_PENDING).length() <= 1 // The former warningsRequireValidCapturedPolicy read the authorization list // the profile kept per verdict; the fact it stated (a warning exactly at a - // confirmed mismatch whose admission captured logging on) lives in - // shadow::recordVerdict and is pinned by - // explicitInequalityRequiresConfirmationTest, + // confirmed mismatch whose admission captured logging on) is + // noWarningWithLoggingOff over the verdict receipts, with + // capturedLoggingFollowsLayers checking the capture itself, and is pinned + // by explicitInequalityRequiresConfirmationTest, // acceptedLoggingSurvivesPolicyChangeTest, // invalidLoggingKeepsComparisonWithoutWarningTest, // omittedLoggingKeepsMismatchMetricsOnlyTest and @@ -483,14 +653,14 @@ module dialcache_shadow_conformance { // Omission and an explicit false reply are distinct environment inputs. Both // keep diagnostic metrics, while neither authorizes a mismatch warning. run omittedLoggingKeepsMismatchMetricsOnlyTest = initialize(2) - .expect(not(s.defaultLog) and not(s.logging)) + .expect(s.defaultLogging == NO_LOGGING_INPUT and s.overlayLogging == NO_LOGGING_INPUT) .then(seedAt(1)).then(beginCall).then(releaseRead).then(resolveAt(VALUE_TWO)) .then(releaseLoad).then(releaseRead) .expect(s.o.shadow == List("mismatch") and s.d.ages == List(0) and s.d.warnings == 0 and s.d.configErrors == 0 and s.o.comparisons == 1) run explicitFalseLoggingOverridesEnabledDefaultTest = initialize(FIRST_LOGGING_FIXTURE + 2) - .expect(s.defaultLog and s.logging) + .expect(s.defaultLogging == LOG_MISMATCHES and s.overlayLogging == NO_LOGGING_INPUT) .then(setLogging(LOGGING_DISABLED)).then(seedAt(1)).then(beginCall) .then(releaseRead).then(resolveAt(VALUE_TWO)).then(releaseLoad).then(releaseRead) .expect(s.o.shadow == List("mismatch") and s.d.ages == List(0) @@ -597,6 +767,8 @@ module dialcache_shadow_conformance { .expect(s.o.calls == List(VALUE_TWO) and s.o.shadow == List("mismatch") and s.o.reads == 2 and s.o.loads == 1 and s.o.writes == 0 and s.d.ages == List(60000) and s.d.warnings == 1 and s.d.futureOffsets == List()) + .expect(s.d.mismatchWarnings == List({ ageMs: 60000, ...LOGGED_EVERYTHING }) + and s.d.loggingOutcomes == List("logged")) // C1 may finish with its retained bytes at the boundary. The next job's C0 // must classify those same stored bytes again and refill instead of decoding. @@ -617,6 +789,7 @@ module dialcache_shadow_conformance { and s.o.reads == 2 and s.o.loads == 1 and s.o.writes == 0 and s.d.ages == List(0) and s.d.warnings == 1 and s.d.futureOffsets == List({ layer: "remote_shadow", offsetMs: 1000 })) + .expect(s.d.mismatchWarnings == List(LOGGED_EVERYTHING) and s.d.loggingOutcomes == List("logged")) run futureDarkAcquisitionReportsOffsetAndFillsMissTest = initialize(0) .then(seedAt(1)).then(rollbackWall).then(beginCall).then(releaseRead) @@ -646,4 +819,145 @@ module dialcache_shadow_conformance { .expect(s.o.shadow == List("mismatch") and s.o.loads == 1 and s.o.writes == 0) .then(beginCall).then(releaseRead).then(resolveAt(VALUE_TWO)) .expect(s.o.calls == List(VALUE_TWO, VALUE_TWO) and s.o.loads == 1 and s.o.dumps == 1) + // -- Mismatch warning content and logging outcomes (C60, D11 to D13, W10's scalar cases) -- + // Each history confirms one mismatch at C0 age 0: text `1` seeded, the + // call, its C0 read, the source's result, the decode and the confirmation + // read of the same bytes. The expected warnings start from WARNING_AT_ZERO, + // which carries neither value field nor diff fields. + action confirmMismatch(value: int): bool = seedAt(1).then(beginCall).then(releaseRead).then(resolveAt(value)) + .then(releaseLoad).then(releaseRead) + pure val WARNING_AT_ZERO: Shadow::MismatchWarning = { + ageMs: 0, cachedValue: Shadow::OMITTED, sourceValue: Shadow::OMITTED, diff: "off", diffNull: false, + count: 0, paths: List(), entries: List(), loggingOutcome: "" + } + pure val ROOT_ONE_TWO: Shadow::Entry = { path: "", cached: VALUE_ONE, source: VALUE_TWO } + pure val LOGGED_EVERYTHING: Shadow::MismatchWarning = { + cachedValue: VALUE_ONE, sourceValue: VALUE_TWO, diff: "values", count: 1, paths: List(""), + entries: List(ROOT_ONE_TWO), ...WARNING_AT_ZERO + } + pure val LOGGED_WITHOUT_VALUES: Shadow::MismatchWarning = + { cachedValue: Shadow::OMITTED, sourceValue: Shadow::OMITTED, ...LOGGED_EVERYTHING } + // A failed construction: the diff fields carried at `values`, all null, the + // cached value field rendered, and the failure it names. + pure def failedWith(sourceValue: int, outcome: str): Shadow::MismatchWarning = + { cachedValue: VALUE_ONE, sourceValue: sourceValue, diff: "values", diffNull: true, loggingOutcome: outcome, + ...WARNING_AT_ZERO } + + // The record's shorthand table, one run per row. Rows 1 and 2 write the + // shorthand and a leaf in one runtime overlay; rows 3 and 4 put a runtime + // shorthand over a default leaf, which it replaces. + run trueShorthandWithValueFalseLogsDiffOnlyTest = initialize(0).then(setLogging(LOGGING_TRUE_WITHOUT_VALUE)) + .then(confirmMismatch(VALUE_TWO)) + .expect(s.o.shadow == List("mismatch") and s.d.configErrors == 0 + and s.d.mismatchWarnings == List(LOGGED_WITHOUT_VALUES) + and s.d.loggingOutcomes == List("logged")) + run falseShorthandWithDiffPathsLogsPathsOnlyTest = initialize(0).then(setLogging(LOGGING_FALSE_WITH_PATHS)) + .then(confirmMismatch(VALUE_TWO)) + .expect(s.o.shadow == List("mismatch") and s.d.configErrors == 0 + and s.d.mismatchWarnings == List({ diff: "paths", count: 1, paths: List(""), ...WARNING_AT_ZERO }) + and s.d.loggingOutcomes == List("logged")) + run runtimeFalseShorthandOverridesDefaultDiffTest = initialize(FIXTURE_DEFAULT_PATHS) + .then(setLogging(LOGGING_DISABLED)).then(confirmMismatch(VALUE_TWO)) + .expect(s.o.shadow == List("mismatch") and s.d.ages == List(0) and s.d.warnings == 0 + and s.d.mismatchWarnings == List() and s.d.loggingOutcomes == List() and s.d.configErrors == 0) + run runtimeTrueShorthandOverridesDefaultValueTest = initialize(FIXTURE_DEFAULT_VALUE_FALSE) + .then(setLogging(LOGGING_ENABLED)).then(confirmMismatch(VALUE_TWO)) + .expect(s.o.shadow == List("mismatch") and s.d.mismatchWarnings == List(LOGGED_EVERYTHING) + and s.d.loggingOutcomes == List("logged")) + + // Each leaf alone logs its own fields; the built-in walk over the model's + // scalars reports nothing for identical renderings (a comparator that calls + // equal values unequal) and one root entry carrying both sides otherwise. + run valueAloneLogsBothRenderingsTest = initialize(0).then(setLogging(LOGGING_VALUE_ALONE)) + .then(confirmMismatch(VALUE_TWO)) + .expect(s.d.mismatchWarnings == List({ cachedValue: VALUE_ONE, sourceValue: VALUE_TWO, ...WARNING_AT_ZERO }) + and s.d.loggingOutcomes == List("logged")) + run valueWithDiffPathsLogsRenderingsAndPathsTest = initialize(FIRST_LOGGING_FIXTURE).then(setLogging(LOGGING_PATHS)) + .then(confirmMismatch(VALUE_TWO)) + .expect(s.d.mismatchWarnings == List({ + cachedValue: VALUE_ONE, sourceValue: VALUE_TWO, diff: "paths", count: 1, paths: List(""), ...WARNING_AT_ZERO + }) + and s.d.loggingOutcomes == List("logged")) + run identicalRenderingsLogNoPathsTest = initialize(FIRST_LOGGING_FIXTURE + 2).then(confirmMismatch(VALUE_ONE)) + .expect(s.o.shadow == List("mismatch") and s.o.comparisons == 1 + and s.d.mismatchWarnings == List({ + cachedValue: VALUE_ONE, sourceValue: VALUE_ONE, diff: "values", ...WARNING_AT_ZERO + }) + and s.d.loggingOutcomes == List("logged")) + run differingScalarsLogOneRootEntryTest = initialize(FIRST_LOGGING_FIXTURE).then(confirmMismatch(VALUE_TWO)) + .expect(s.o.shadow == List("mismatch") and s.d.mismatchWarnings == List(LOGGED_EVERYTHING) + and s.d.loggingOutcomes == List("logged")) + + // Each failure outcome alone and combined (D11, D15): the throwing diff + // hook (fixture 13), a source result with no JSON rendering (D13) and a + // logger that throws. `hook_error` outranks `render_error`, which outranks + // `logger_error`; a logger failure leaves the warning's content as logged. + run hookErrorNullsDiffFieldsTest = initialize(FIXTURE_DIFF_HOOK).then(confirmMismatch(VALUE_TWO)) + .expect(s.o.shadow == List("mismatch") and s.o.calls == List(VALUE_TWO) + and s.d.mismatchWarnings == List(failedWith(VALUE_TWO, "hook_error")) + and s.d.loggingOutcomes == List("hook_error")) + run renderErrorNullsUnrenderableSourceTest = initialize(FIRST_LOGGING_FIXTURE) + .then(confirmMismatch(VALUE_UNRENDERABLE)) + .expect(s.o.shadow == List("mismatch") and s.o.calls == List(VALUE_UNRENDERABLE) + and s.d.mismatchWarnings == List(failedWith(Shadow::NO_RENDERING, "render_error")) + and s.d.loggingOutcomes == List("render_error")) + run loggerErrorStillDeliversWarningTest = initialize(FIRST_LOGGING_FIXTURE).then(loggerFaultAt(1)) + .then(confirmMismatch(VALUE_TWO)) + .expect(s.o.shadow == List("mismatch") and s.d.mismatchWarnings == List(LOGGED_EVERYTHING) + and s.d.loggingOutcomes == List("logger_error")) + run hookErrorOutranksRenderErrorTest = initialize(FIXTURE_DIFF_HOOK).then(confirmMismatch(VALUE_UNRENDERABLE)) + .expect(s.d.mismatchWarnings == List(failedWith(Shadow::NO_RENDERING, "hook_error")) + and s.d.loggingOutcomes == List("hook_error")) + run hookErrorOutranksLoggerErrorTest = initialize(FIXTURE_DIFF_HOOK).then(loggerFaultAt(1)) + .then(confirmMismatch(VALUE_TWO)) + .expect(s.d.mismatchWarnings == List(failedWith(VALUE_TWO, "hook_error")) + and s.d.loggingOutcomes == List("hook_error")) + run renderErrorOutranksLoggerErrorTest = initialize(FIRST_LOGGING_FIXTURE).then(loggerFaultAt(1)) + .then(confirmMismatch(VALUE_UNRENDERABLE)) + .expect(s.d.mismatchWarnings == List(failedWith(Shadow::NO_RENDERING, "render_error")) + and s.d.loggingOutcomes == List("render_error")) + run hookErrorOutranksBothFailuresTest = initialize(FIXTURE_DIFF_HOOK).then(loggerFaultAt(1)) + .then(confirmMismatch(VALUE_UNRENDERABLE)) + .expect(s.d.mismatchWarnings == List(failedWith(Shadow::NO_RENDERING, "hook_error")) + and s.d.loggingOutcomes == List("hook_error")) + // The hook runs only when `diff` is above off: with `value` alone the + // unrenderable source is the failure the warning names. + run hookSkippedWithDiffOffTest = initialize(FIXTURE_DIFF_HOOK).then(setLogging(LOGGING_VALUE_ALONE)) + .then(confirmMismatch(VALUE_UNRENDERABLE)) + .expect(s.d.mismatchWarnings == List({ + cachedValue: VALUE_ONE, sourceValue: Shadow::NO_RENDERING, loggingOutcome: "render_error", ...WARNING_AT_ZERO + }) + and s.d.loggingOutcomes == List("render_error")) + + // An invalid input acts off and is reported once; a valid leaf beside it + // stays on, whether the invalid input is the other leaf or the shorthand. + run invalidDiffLeafKeepsValueLeafTest = initialize(0).then(setLogging(LOGGING_INVALID_DIFF)) + .then(confirmMismatch(VALUE_TWO)) + .expect(s.d.configErrors == 1 + and s.d.mismatchWarnings == List({ cachedValue: VALUE_ONE, sourceValue: VALUE_TWO, ...WARNING_AT_ZERO }) + and s.d.loggingOutcomes == List("logged")) + run invalidValueLeafKeepsDiffLeafTest = initialize(FIRST_LOGGING_FIXTURE).then(setLogging(LOGGING_INVALID_VALUE)) + .then(confirmMismatch(VALUE_TWO)) + .expect(s.d.configErrors == 1 + and s.d.mismatchWarnings == List({ diff: "paths", count: 1, paths: List(""), ...WARNING_AT_ZERO }) + and s.d.loggingOutcomes == List("logged")) + run invalidShorthandKeepsDiffLeafBesideItTest = initialize(0).then(setLogging(LOGGING_INVALID_SHORTHAND_WITH_VALUES)) + .then(confirmMismatch(VALUE_TWO)) + .expect(s.d.configErrors == 1 + and s.d.mismatchWarnings == List(LOGGED_WITHOUT_VALUES) + and s.d.loggingOutcomes == List("logged")) + + // A leaf the runtime overlay writes off is a written value, not an unwritten + // one: over the `logMismatches: true` default it replaces its own inherited + // leaf, and the other inherited leaf stays on. + run runtimeOffDiffOverridesDefaultDiffTest = initialize(FIRST_LOGGING_FIXTURE).then(setLogging(LOGGING_DIFF_OFF)) + .then(confirmMismatch(VALUE_TWO)) + .expect(s.o.shadow == List("mismatch") and s.d.configErrors == 0 + and s.d.mismatchWarnings == List({ cachedValue: VALUE_ONE, sourceValue: VALUE_TWO, ...WARNING_AT_ZERO }) + and s.d.loggingOutcomes == List("logged")) + run runtimeFalseValueOverridesDefaultValueTest = initialize(FIRST_LOGGING_FIXTURE) + .then(setLogging(LOGGING_VALUE_FALSE)).then(confirmMismatch(VALUE_TWO)) + .expect(s.o.shadow == List("mismatch") and s.d.configErrors == 0 + and s.d.mismatchWarnings == List(LOGGED_WITHOUT_VALUES) + and s.d.loggingOutcomes == List("logged")) } diff --git a/formal/models/dialcache-shadow-layers-conformance.qnt b/formal/models/dialcache-shadow-layers-conformance.qnt index 80914755..7ab6d798 100644 --- a/formal/models/dialcache-shadow-layers-conformance.qnt +++ b/formal/models/dialcache-shadow-layers-conformance.qnt @@ -141,7 +141,10 @@ module dialcache_shadow_layers_conformance { not(ordinaryStaleCandidate(choice % KEYS)), input' = { name: "beginCall", choice: choice }, s' = Shadow::begin(s, LAYOUT, call(choice), s.resolved, s.ttls, Recovery::ALLOW, - { hook: true, selected: s.shadow, capacity: JOB_CAPACITY, budgetMs: JOB_BUDGET_MS, logging: false }) + { + hook: true, selected: s.shadow, capacity: JOB_CAPACITY, budgetMs: JOB_BUDGET_MS, + logging: Shadow::LOGGING_OFF, diffHook: false + }) } action beginCall = { nondet choice = 0.to(14).oneOf() diff --git a/formal/models/dialcache-shadow-read-deadlines-conformance.qnt b/formal/models/dialcache-shadow-read-deadlines-conformance.qnt index 803bc30a..522e177c 100644 --- a/formal/models/dialcache-shadow-read-deadlines-conformance.qnt +++ b/formal/models/dialcache-shadow-read-deadlines-conformance.qnt @@ -31,10 +31,16 @@ module dialcache_shadow_read_deadlines_conformance { } pure val DARK: Resolution = { layers: { request: false, local: false, remote: false }, coalesce: true } pure val TTLS: Ttls = { localMs: 0, freshMs: TTL, retentionMs: TTL } - pure val SHADOW: Shadow::Shadowing = { hook: true, selected: true, capacity: 1, budgetMs: SOURCE_BUDGET, logging: false } + pure val SHADOW: Shadow::Shadowing = { + hook: true, selected: true, capacity: 1, budgetMs: SOURCE_BUDGET, logging: Shadow::LOGGING_OFF, diffHook: false + } pure def call(key: int): Call = { instance: 0, key: key, context: 0, enabled: true, keyFailed: false } - type Channel = { configErrors: int, warnings: int, ages: List[int], coalesced: List[str], fallbackErrors: List[str], futureOffsets: List[Diagnostics::FutureOffset] } + type Channel = { + configErrors: int, warnings: int, ages: List[int], coalesced: List[str], fallbackErrors: List[str], + futureOffsets: List[Diagnostics::FutureOffset], + mismatchWarnings: List[Shadow::MismatchWarning], loggingOutcomes: List[str] + } type State = { o: Observation, d: Channel, io: RemoteIO::IO, now: int, skew: int, remoteAvailable: bool, tracked: bool, capacity: int, @@ -53,7 +59,10 @@ module dialcache_shadow_read_deadlines_conformance { action init = all { input' = { name: "init", choice: -1 }, s' = { - o: empty, d: { configErrors: 0, warnings: 0, ages: List(), coalesced: List(), fallbackErrors: List(), futureOffsets: List() }, io: RemoteIO::NO_IO, + o: empty, d: { + configErrors: 0, warnings: 0, ages: List(), coalesced: List(), fallbackErrors: List(), futureOffsets: List(), + mismatchWarnings: List(), loggingOutcomes: List() + }, io: RemoteIO::NO_IO, now: 0, skew: 100000, remoteAvailable: true, tracked: true, capacity: 0, readFailed: false, dumpFailed: false, writeFailed: false, readBudget: 5, closed: List(), memo: List(), requestFlights: List(), processFlights: List(NO_OWNER, NO_OWNER), diff --git a/formal/models/fixtures/kernel/held-dark-shadow.qnt b/formal/models/fixtures/kernel/held-dark-shadow.qnt index ac181467..113f97f5 100644 --- a/formal/models/fixtures/kernel/held-dark-shadow.qnt +++ b/formal/models/fixtures/kernel/held-dark-shadow.qnt @@ -28,13 +28,19 @@ module held_dark_shadow { pure val LAYOUT: Serving::Layout = { keysPerInstance: 2, keysPerScope: 2, persistentContexts: 0, operationsPerEntity: 1, probeSourceScope: false, policyProvider: true } pure val DARK: Resolution = { layers: { request: false, local: false, remote: false }, coalesce: false } pure val TTLS: Ttls = { localMs: 0, freshMs: 1000, retentionMs: 5000 } - pure val SHADOWING: Shadow::Shadowing = { hook: true, selected: true, capacity: 1, budgetMs: 100, logging: false } + pure val SHADOWING: Shadow::Shadowing = { + hook: true, selected: true, capacity: 1, budgetMs: 100, logging: Shadow::LOGGING_OFF, diffHook: false + } // The same policy with a job budget above the source budget and two slots. pure val LONG_SHADOWING: Shadow::Shadowing = { capacity: 2, budgetMs: 200, ...SHADOWING } pure def call(key: int): Call = { instance: 0, key: key, context: 0, enabled: true, keyFailed: false } type State = { - o: Observation, d: { configErrors: int, warnings: int, ages: List[int], coalesced: List[str], fallbackErrors: List[str], futureOffsets: List[Diagnostics::FutureOffset] }, + o: Observation, d: { + configErrors: int, warnings: int, ages: List[int], coalesced: List[str], fallbackErrors: List[str], + futureOffsets: List[Diagnostics::FutureOffset], + mismatchWarnings: List[Shadow::MismatchWarning], loggingOutcomes: List[str] + }, io: RemoteIO::IO, now: int, skew: int, remoteAvailable: bool, tracked: bool, capacity: int, readFailed: bool, dumpFailed: bool, writeFailed: bool, closed: List[bool], memo: List[int], requestFlights: List[int], processFlights: List[int], @@ -51,7 +57,10 @@ module held_dark_shadow { action init = all { input' = { name: "init", choice: -1 }, s' = { - o: empty, d: { configErrors: 0, warnings: 0, ages: List(), coalesced: List(), fallbackErrors: List(), futureOffsets: List() }, + o: empty, d: { + configErrors: 0, warnings: 0, ages: List(), coalesced: List(), fallbackErrors: List(), futureOffsets: List(), + mismatchWarnings: List(), loggingOutcomes: List() + }, io: RemoteIO::NO_IO, now: 0, skew: 100000, remoteAvailable: true, tracked: true, capacity: 0, readFailed: false, dumpFailed: false, writeFailed: false, closed: List(), memo: List(), requestFlights: List(), processFlights: List(NO_OWNER, NO_OWNER), diff --git a/formal/models/fixtures/kernel/held-shadow.qnt b/formal/models/fixtures/kernel/held-shadow.qnt index 280e67e0..483e9cf4 100644 --- a/formal/models/fixtures/kernel/held-shadow.qnt +++ b/formal/models/fixtures/kernel/held-shadow.qnt @@ -35,7 +35,9 @@ module held_shadow { pure val LAYOUT: Serving::Layout = { keysPerInstance: 2, keysPerScope: 2, persistentContexts: 1, operationsPerEntity: 1, probeSourceScope: true, policyProvider: true } pure val REMOTE: Resolution = { layers: { request: false, local: false, remote: true }, coalesce: true } pure val TTLS: Ttls = { localMs: 0, freshMs: 1000, retentionMs: 5000 } - pure val SHADOWING: Shadow::Shadowing = { hook: true, selected: true, capacity: 1, budgetMs: 100, logging: false } + pure val SHADOWING: Shadow::Shadowing = { + hook: true, selected: true, capacity: 1, budgetMs: 100, logging: Shadow::LOGGING_OFF, diffHook: false + } pure val READ_BUDGET = 50 pure def call(key: int): Call = { instance: 0, key: key, context: 1, enabled: true, keyFailed: false } // A call inside the one persistent scope, the one a run can hold a reply for. diff --git a/formal/models/fixtures/kernel/shadow-jobs.qnt b/formal/models/fixtures/kernel/shadow-jobs.qnt index f90c34ed..29be6517 100644 --- a/formal/models/fixtures/kernel/shadow-jobs.qnt +++ b/formal/models/fixtures/kernel/shadow-jobs.qnt @@ -32,8 +32,12 @@ module shadow_jobs { pure val SERVING: Resolution = { layers: { request: false, local: false, remote: true }, coalesce: false } pure val DARK: Resolution = { layers: { request: false, local: false, remote: false }, coalesce: false } pure val TTLS: Ttls = { localMs: 0, freshMs: 1000, retentionMs: 5000 } - pure val SHADOWING: Shadow::Shadowing = { hook: true, selected: true, capacity: 1, budgetMs: 100, logging: false } - pure val UNSELECTED: Shadow::Shadowing = { hook: true, selected: false, capacity: 1, budgetMs: 100, logging: false } + pure val SHADOWING: Shadow::Shadowing = { + hook: true, selected: true, capacity: 1, budgetMs: 100, logging: Shadow::LOGGING_OFF, diffHook: false + } + pure val UNSELECTED: Shadow::Shadowing = { + hook: true, selected: false, capacity: 1, budgetMs: 100, logging: Shadow::LOGGING_OFF, diffHook: false + } pure def call(key: int): Call = { instance: 0, key: key, context: 1, enabled: true, keyFailed: false } // A call inside the one persistent scope, the scope a run can close. pure def scopedCall(key: int): Call = { instance: 0, key: key, context: 0, enabled: true, keyFailed: false } @@ -233,8 +237,8 @@ module shadow_jobs { // A budget delivered inside an advance is delivered at its own instant, in // registration order, before the clock reaches the target. run budgetsDeliverInRegistrationOrderTest = init - .then(beginWith(0, DARK, Recovery::ALLOW, { hook: true, selected: true, capacity: 2, budgetMs: 100, logging: false })).then(advanceBy(10)) - .then(beginWith(1, DARK, Recovery::ALLOW, { hook: true, selected: true, capacity: 2, budgetMs: 100, logging: false })) + .then(beginWith(0, DARK, Recovery::ALLOW, { capacity: 2, ...SHADOWING })).then(advanceBy(10)) + .then(beginWith(1, DARK, Recovery::ALLOW, { capacity: 2, ...SHADOWING })) .expect(s.deadlines.length() == 2) .then(advanceBy(1000)) .expect(s.o.shadow == List("timeout", "timeout") and s.now == 1010 and s.deadlines == List() diff --git a/formal/models/kernel/README.md b/formal/models/kernel/README.md index 115dd20c..4e2ec235 100644 --- a/formal/models/kernel/README.md +++ b/formal/models/kernel/README.md @@ -27,6 +27,7 @@ codes are omitted; the module header lists them. | `runtime_policy` | How a runtime policy reply (the runtime-boundaries drivers' codes 0 to 21) resolves against an instance's configured baseline: serving cohorts, omitted, null and invalid leaves, runtime TTLs, the kill switch. | `resolve` | | `policy_overlay` | How a runtime policy overlay (the policy drivers' codes 0 to 25) resolves over configured TTLs: each layer's TTL, the refill retention, coalescing and whether the reply failed (C13, C17, C20 to C22). | `resolve`, `ttls`, `failed`, `localTtl`, `remoteTtl`, `retention` | | `config_errors` | The policy-error channel: a reply that fails to resolve is reported once, against no layer, as a configuration error (C20); an opt-in record the traversal never reads. | `recordConfigError` | +| `mismatch_logging` | How a shadow policy's mismatch-logging inputs, one config layer each, resolve to the leaves a shadow job captures (C60): each layer's shorthand expanded, the runtime overlay merged over the defaults leaf by leaf, each leaf resolved on its own and one configuration error for every invalid input. Only the shadow profile composes it; `shadow` never imports it. | `expandLogging`, `mergeLogging`, `resolveLogging` | | `request_memo` | Request-scoped memo rows and scope lifetime: a persistent context owns one row of slots; closing the scope forgets the row, and a late settlement can no longer memoize into it. | `openScope`, `closeScope`, `memoize`, `memoSlot`, `memoValue`, `scopeOpen` | | `local_storage` | Per-instance local storage with LRU eviction and strict insertion expiry; zero capacity stores nothing and never fails; an expired entry keeps its slot until replaced or evicted. | `localValue`, `promote`, `putLocal` | | `local_grid` | The local projection with the raw environment instants kept beside the ordinary expiry, so a precise-clock expiry error is executable instead of hidden by rounding (C09). | `call` | @@ -45,7 +46,7 @@ codes are omitted; the module header lists them. | `remote_writes` | The atomic remote lifecycle with held writes: a refill serializes when its source settles accepted (the dump), then dispatches the write, then stores the frame; a fault at either step stops the rest. | `releaseJudged`, `settle`, `settleLoader`, `holdDump`, `dispatch`, `releaseDump`, `store`, `releaseWrite`, `abandonDump`, `abandonWrite`, `complete`, `finish`, `failed`, `latestDump`, `latestWrite` | | `adapter_replies` | The queued adapter reply (C55): what the adapter returns in place of the stored frame, interpreted once at the trust boundary; the drivers queue one reply by its class. | `queue`, `consume`, `queued`, `isFrame`, `carriesFence`, `fenceOf`, `reason`, `observed`, `settleRead` | | `diagnostics` | The diagnostics channel: which singleflight a caller coalesced into, the layer a failed source is attributed to, the recovery age, warnings and future offsets, over the request-only and the held lifecycles. | `admitRequest`, `releaseRequest`, `settleRequest`; `admit`, `releaseJudged`, `begin`, `beginRead`, `settleLoad`, `settleLoader`, `expireLoader`, `advance`; `recordAge`, `recordWarning`, `recordConfigError`, `recordFutureOffset` | -| `shadow` | Shadow jobs (C47 to C54, C57, C60), admitted beside the caller in one bounded per-instance registry: a served hit's job checks the served payload against a detached source, a dark job's C0 read against the caller's own source. | `begin`, `release`, `settle`, `advance`; served jobs: `admitServed`, `settleJob`, `fill`, `confirmation`, `finishJob`, `expireJob`; held: `beginHeld`, `releaseHeld`, `settleHeld`, `settleRead`, `settleLoad`, `settleLoader`, `expireHeld`, `advanceHeld`; dark: `admitDark`, `dispatchC0`, `beginDark`, `recordVerdict`, `releaseRead`, `releaseLoad`, `releaseDump`, `releaseWrite` | +| `shadow` | Shadow jobs (C47 to C54, C57, C60), admitted beside the caller in one bounded per-instance registry: a served hit's job checks the served payload against a detached source, a dark job's C0 read against the caller's own source; a job captures the mismatch-logging leaves `mismatch_logging` resolves, and a confirmed mismatch's warning content and logging outcome are judged at the verdict over scalar values (W10's root cases). | `begin`, `release`, `settle`, `advance`; served jobs: `admitServed`, `settleJob`, `fill`, `confirmation`, `finishJob`, `expireJob`; held: `beginHeld`, `releaseHeld`, `settleHeld`, `settleRead`, `settleLoad`, `settleLoader`, `expireHeld`, `advanceHeld`; dark: `admitDark`, `dispatchC0`, `beginDark`, `recordVerdict`, `releaseRead`, `releaseReadWithLogger`, `releaseLoad`, `releaseDump`, `releaseWrite`; logging: `warningOf`, `loggingOutcomeOf`, `recordMismatchWarning`, `recordVerdictInputs` | | `metrics` | The observer's metric events over the remote layer (C30, C57 to C59): the ordered trail the implementations report to their metrics callback, one event per read, write, miss reason and error. | `event`, `begin`, `settleRead`, `settleLoad`, `settleLoader`, `sourceDone`, `expireLoader`, `expireRead`, `advance`, `releaseDump`, `releaseWrite`, `invalidate`, `missReason`, `futureLead` | | `shadow_reads` | A shadow read's two completions (C54, C56): its bounded result ends at the read deadline while the raw adapter operation keeps the slot until it settles; the job deadline is independent. | `begin`, `releaseRead`, `releaseLoad`, `advance` | @@ -57,7 +58,7 @@ at `serving` and read outward, which is how a rule is usually traced. | Layer | Modules | What they hold | | --- | --- | --- | -| 0 | `encodings`, `calls`, `clock`, `layer_policy`, `config_errors`, `instances` | Vocabulary: sentinels, the call record, time, the immediate policy reply, the policy-error record and which instances exist. No rule reads another kernel module here. | +| 0 | `encodings`, `calls`, `clock`, `layer_policy`, `config_errors`, `instances`, `mismatch_logging` | Vocabulary: sentinels, the call record, time, the immediate policy reply, the policy-error record, which instances exist and the mismatch-logging inputs. No rule reads another kernel module here. | | 1 | `request_memo`, `local_storage`, `payloads`, `policy_gate`, `flights`, `policy_overlay`, `runtime_policy` | The storage and registries each serving layer owns, the payload classes, the held-reply gate and the two runtime reply decoders. | | 2 | `remote_frames` | The remote layer's storage: frames with stamps, retention and watermarks. | | 3 | `serving`, `recovery` | The traversal and publication rules, and the stale-on-error judgments the held and atomic lifecycles consult. Start here to trace a rule. | @@ -162,7 +163,10 @@ The composed profiles are, in order: the dark arm of the held shadow lifecycle. The dark job's C0 read, decode and confirmation read are held in `remote_io`'s lists and released by ordinal over the diagnosed budgeted held remote lifecycle, with the held - fill and the buffered invalidation. + fill and the buffered invalidation. It keeps its logging inputs as + written, resolves them at admission through `mismatch_logging`, and + composes the verdict receipts its warning properties read + (`recordVerdictInputs`). 14. `formal/models/dialcache-effects-conformance.qnt` is the fourteenth, over the held remote lifecycle with its refills held, the queued adapter reply and the metric events, the first whose drivers' channel is an ordered event diff --git a/formal/models/kernel/mismatch-logging.qnt b/formal/models/kernel/mismatch-logging.qnt new file mode 100644 index 00000000..6c3a1412 --- /dev/null +++ b/formal/models/kernel/mismatch-logging.qnt @@ -0,0 +1,68 @@ +module mismatch_logging { + // How the mismatch-logging inputs a shadow policy writes resolve to the leaves a job captures (C60). + // + // Rules + // - Each config layer is expanded on its own: the `logMismatches` shorthand stands for both + // leaves, and a leaf written beside it overrides what it stands for (`expandLogging`). + // - The runtime overlay is merged over the defaults leaf by leaf (`mergeLogging`). + // - Each leaf resolves on its own: an unwritten leaf is off, a value of the wrong type acts off, + // and one configuration error covers every invalid input (`resolveLogging`). + // - The result is the `{ value, diff }` record `shadow::Logging` names: a profile resolves its + // inputs here and hands the record to the job at admission, so `shadow` never imports this. + // Shapes + // - LoggingInput: one config layer's inputs as written, before expansion. + // Consumers: shadow. + + // The diff levels, spelled as `shadow` spells them: the drivers' encoding + // of `mismatchLogging.diff`, in order. + pure val DIFF_OFF = "off" + pure val DIFF_PATHS = "paths" + pure val DIFF_VALUES = "values" + // One config layer's inputs as written, before expansion: the + // `logMismatches` shorthand and the two `mismatchLogging` leaves, each + // UNWRITTEN, the spelling of a valid value ("true", "false", a level) or + // any other string, a value of the wrong type. + pure val UNWRITTEN = "" + type LoggingInput = { shorthand: str, value: str, diff: str } + pure val NO_LOGGING_INPUT: LoggingInput = { shorthand: UNWRITTEN, value: UNWRITTEN, diff: UNWRITTEN } + // A layer expanded (D10): its shorthand stands for leaves, `true` for + // `{ value: true, diff: "values" }` and `false` for both off, while a + // malformed one stands for both off and is marked invalid; a leaf written + // beside the shorthand then overrides what it stands for. + pure def expandLogging(written: LoggingInput): { value: str, diff: str, invalid: bool } = { + val stands = if (written.shorthand == "true") { value: "true", diff: DIFF_VALUES } + else if (written.shorthand == UNWRITTEN) { value: UNWRITTEN, diff: UNWRITTEN } + else { value: "false", diff: DIFF_OFF } + { + value: if (written.value != UNWRITTEN) written.value else stands.value, + diff: if (written.diff != UNWRITTEN) written.diff else stands.diff, + invalid: not(Set(UNWRITTEN, "true", "false").contains(written.shorthand)) + } + } + // The runtime overlay over the defaults, leaf by leaf, each layer expanded + // first: a runtime shorthand replaces both inherited leaves, and a leaf the + // overlay leaves unwritten falls through to the default's. Registration + // rejects a malformed static input, so only the overlay marks the merge. + pure def mergeLogging(defaults: LoggingInput, overlay: LoggingInput): { value: str, diff: str, invalid: bool } = { + val inherited = expandLogging(defaults) + val written = expandLogging(overlay) + { + value: if (written.value != UNWRITTEN) written.value else inherited.value, + diff: if (written.diff != UNWRITTEN) written.diff else inherited.diff, + invalid: written.invalid + } + } + // Each leaf resolved on its own: an unwritten leaf is off and a value of the + // wrong type acts off, so an invalid leaf beside a valid one leaves the + // valid one on; one configuration error covers every invalid input. + pure def resolveLogging(merged: { value: str, diff: str, invalid: bool }): { + logging: { value: bool, diff: str }, invalid: bool + } = { + val levels = Set(DIFF_OFF, DIFF_PATHS, DIFF_VALUES) + { + logging: { value: merged.value == "true", diff: if (levels.contains(merged.diff)) merged.diff else DIFF_OFF }, + invalid: merged.invalid or not(Set(UNWRITTEN, "true", "false").contains(merged.value)) + or not(levels.contains(merged.diff) or merged.diff == UNWRITTEN) + } + } +} diff --git a/formal/models/kernel/shadow.qnt b/formal/models/kernel/shadow.qnt index ed28b255..a5a705c7 100644 --- a/formal/models/kernel/shadow.qnt +++ b/formal/models/kernel/shadow.qnt @@ -9,18 +9,21 @@ module shadow { // job with no C0 fills, under that authority, the fence judged again at dispatch (C52, C53). // - A C0 is decoded on its own and compared; an unequal comparison needs one confirmation read // of the frame's bytes (C50, C51); verdict ages and mismatch warnings follow C57 and C60. + // - A job captures the mismatch-logging leaves; a confirmed mismatch with either on warns once, + // its content and logging outcome judged at the verdict (C60; W10's scalar cases). // Shapes // - Comparison: the comparator the drivers install and the elapsed time it consumes. // - Shadowing: the shadow policy in force for a call, passed at admission. - // - Malformed: whether the ramp or the logging value failed to resolve, passed to `beginDark`. + // - Malformed: whether the ramp or a logging input failed to resolve, passed to `beginDark`. + // - Logging: the leaves a job captures, resolved from the written inputs by `mismatch_logging`. // - Job: one job's record, kept while it is pending. // - Jobs[r]: the per-instance registry of bounded capacity both kinds share, over the clock. // - Shadowed[r]: the atomic lifecycle over `remote_writes::Writing`, shadow-layers' shape. // - HeldJobs[r]: the held lifecycle over `remote_io::Held`, with the held fills. // - HeldShadowed[r]: the served arm (admission profile), the selection captured per flight. // - DiagnosedShadowed[r, q]: the dark arm (shadow profile) over `diagnostics::DiagnosedHeld`. - // Consumers: admission, dark-layers, shadow, shadow-layers, shadow-read-deadlines; also - // composed through `shadow_reads`. + // - Entry, MismatchWarning, Verdict: a warning's content, and a verdict's opt-in receipt. + // Consumers: admission, dark-layers, shadow, shadow-layers, shadow-read-deadlines, `shadow_reads`. import conformance_observations.* from "../conformance-observations" import encodings.* from "./encodings" import calls.* from "./calls" @@ -54,16 +57,28 @@ module shadow { type Comparison = { comparator: int, elapsedMs: int } pure val VALUE_COMPARISON: Comparison = { comparator: COMPARE_VALUES, elapsedMs: 0 } + // -- Mismatch logging (C60) ---------------------------------------------------------- + // The leaves a confirmed-mismatch warning is logged under: `value` logs both + // renderings, and the ordered `diff` level logs the difference's paths + // (`paths`) or its entries too (`values`). Either on opts into a warning. + pure val DIFF_OFF = "off" + pure val DIFF_PATHS = "paths" + pure val DIFF_VALUES = "values" + type Logging = { value: bool, diff: str } + pure val LOGGING_OFF: Logging = { value: false, diff: DIFF_OFF } + pure def logs(logging: Logging): bool = logging.value or logging.diff != DIFF_OFF + // The shadow policy in force for a call: whether the outcome hook is // installed (C47), whether the key's cohort is selected, the per-instance - // capacity, the job budget and whether a confirmed mismatch is logged (C60, - // captured by the job at admission). Cohort selection is the drivers' ramp - // decoded by the profile through `cohort_boundaries` (strict against the - // key's numerator); the module reads the decoded selection. - type Shadowing = { hook: bool, selected: bool, capacity: int, budgetMs: int, logging: bool } - // Whether the policy failed to resolve: the ramp, or the logging value. A + // capacity, the job budget, the mismatch-logging leaves (C60) and whether a + // `shadowMismatchDiff` hook replaces the built-in walk (D6; the model's hook + // throws), both captured by the job at admission. Cohort selection is the + // drivers' ramp decoded by the profile through `cohort_boundaries` (strict + // against the key's numerator); the module reads the decoded selection. + type Shadowing = { hook: bool, selected: bool, capacity: int, budgetMs: int, logging: Logging, diffHook: bool } + // Whether the policy failed to resolve: the ramp, or a logging input. A // shadow policy that fails to resolve is one configuration error per call: - // an invalid ramp always; an invalid logging value only where the hook and + // an invalid ramp always; any invalid logging input only where the hook and // the ramp would otherwise admit the job (`beginDark`). type Malformed = { ramp: bool, logging: bool } pure val WELL_FORMED: Malformed = { ramp: false, logging: false } @@ -73,10 +88,14 @@ module shadow { // dark, the payload its C0 found fresh (NO_VALUE when none) and that // frame's stamp (the age a verdict samples), the fence its C0 read observed // when it found no visible frame (C53), the TTLs its reply resolved to - // (C18), its phase, whether its budget expired and the mismatch logging it - // captured. The captured TTLs are the freshness its C0 read judges by and - // the retention its fill is written with, whatever the policy says later. - type Job = { source: int, dark: bool, cached: int, stamp: int, fence: int, ttls: Ttls, phase: int, timedOut: bool, logging: bool } + // (C18), its phase, whether its budget expired, and the mismatch logging + // and diff hook it captured. The captured TTLs are the freshness its C0 + // read judges by and the retention its fill is written with, whatever the + // policy says later. + type Job = { + source: int, dark: bool, cached: int, stamp: int, fence: int, ttls: Ttls, phase: int, timedOut: bool, + logging: Logging, diffHook: bool + } // The registry beside the clock and its pending budgets, over the sources // the jobs run for: what every lifecycle composes. The registry is // pending-only, keyed by the job's source: a finished job leaves no record, @@ -113,10 +132,16 @@ module shadow { // (`deadlines`, `diagnostics`: the caller's source deadline, the coalesced // scope, the layer a failure is attributed to) with the held fills and the // jobs. Beside the four channel fields every descriptor parses, it carries - // the two the shadow drivers compare. + // the four the shadow drivers compare: the configuration errors, the future + // offsets, the mismatch warnings with their content and one logging outcome + // per warning (`recordMismatchWarning`); a profile whose drivers never log a + // mismatch carries the last two empty. type DiagnosedShadowed[r, q] = Diagnostics::DiagnosedHeld[ { dumps: List[Writes::HeldDump], writes: List[Writes::HeldWrite], jobs: List[Job] | r }, - { configErrors: int, futureOffsets: List[Diagnostics::FutureOffset] | q } + { + configErrors: int, futureOffsets: List[Diagnostics::FutureOffset], + mismatchWarnings: List[MismatchWarning], loggingOutcomes: List[str] | q + } ] pure def holdsJob(state: Jobs[r], source: int): bool = state.jobs.select(job => job.source == source).length() > 0 @@ -158,8 +183,12 @@ module shadow { pure def admits(shadowing: Shadowing): bool = shadowing.hook and shadowing.selected pure def dropped(state: Jobs[r]): Jobs[r] = { o: { shadow: state.o.shadow.append("dropped"), ...state.o }, ...state } - pure def newJob(source: int, dark: bool, cached: int, stamp: int, fence: int, ttls: Ttls, phase: int, logging: bool): Job = - { source: source, dark: dark, cached: cached, stamp: stamp, fence: fence, ttls: ttls, phase: phase, timedOut: false, logging: logging } + // A pending job, the shadow policy's mismatch logging and diff hook captured. + pure def newJob(source: int, dark: bool, cached: int, stamp: int, fence: int, ttls: Ttls, phase: int, + shadowing: Shadowing): Job = { + source: source, dark: dark, cached: cached, stamp: stamp, fence: fence, ttls: ttls, phase: phase, timedOut: false, + logging: shadowing.logging, diffHook: shadowing.diffHook + } // A job registered over its source, its budget due from now: every job has // the budget its admission resolved, kept by `deadlines` (kind `JOB`, keyed // by the job's source). @@ -182,7 +211,9 @@ module shadow { if (not(admits(shadowing))) state else if (busy(state, who.instance, who.key, shadowing.capacity)) dropped(state) else admit(Serving::startDetachedSource(state, layout, who), - newJob(state.sources.length(), false, cached, state.created.nth(who.key), NO_FENCE, ttls, WAITING_SOURCE, shadowing.logging), shadowing.budgetMs) + newJob(state.sources.length(), false, cached, state.created.nth(who.key), NO_FENCE, ttls, WAITING_SOURCE, + shadowing), + shadowing.budgetMs) // Admission beside the release judged once (`serving::layeredRelease`), on // the released state: the atomic lifecycle, where a dark job's C0 read is // made here. A served hit admits a served job (`admitServed`); a miss whose @@ -213,7 +244,8 @@ module shadow { if (c0.failed) { o: { shadow: read.o.shadow.append("redis_error"), ...read.o }, ...read } else admit(read, newJob(state.sources.length() - 1, true, acquired(c0), state.created.nth(call.key), - Remote::fenceFor(state, call.key, layout.operationsPerEntity, c0.visible), ttls, WAITING_SOURCE, shadowing.logging), + Remote::fenceFor(state, call.key, layout.operationsPerEntity, c0.visible), ttls, WAITING_SOURCE, + shadowing), shadowing.budgetMs) } | _ => state @@ -486,7 +518,8 @@ module shadow { else match judged.decision.outcome { | Starts => if (not(judged.open) or judged.effective.layers.remote) state else if (busy(state, call.instance, call.key, shadowing.capacity)) dropped(state) - else admit(state, newJob(state.sources.length() - 1, true, NO_VALUE, 0, NO_FENCE, ttls, READING_C0, shadowing.logging), shadowing.budgetMs) + else admit(state, newJob(state.sources.length() - 1, true, NO_VALUE, 0, NO_FENCE, ttls, READING_C0, shadowing), + shadowing.budgetMs) // The remote layer is unobserved at a held release: nothing serves from // it here, so no served job is admitted at this step. | _ => state @@ -531,37 +564,127 @@ module shadow { settleJobHeld(settled, layout, flight) else settled } + // -- Mismatch warnings (C60, D11 to D13) ---------------------------------------------- + + // A side of a warning's value fields or of an entry: OMITTED when the + // warning or the entry leaves it out, NO_RENDERING when it is logged as + // `null` (a value with no JSON rendering, D13), else the value's code (a + // source's result; a C0's decoded value, `payloads::valueOf`). + pure val OMITTED = -1 + pure val NO_RENDERING = 0 + // One difference the walk reports: its RFC 6901 pointer and both sides. + type Entry = { path: str, cached: int, source: int } + // One warning's content as the drivers parse it (D12): the age of its C0 at + // the verdict, the two value fields, which `diff` fields it carries and + // whether they are `null`, the uncut count, the paths, the entries (logged + // at `values` only) and the construction failure it names ("" for none). + type MismatchWarning = { + ageMs: int, cachedValue: int, sourceValue: int, diff: str, diffNull: bool, + count: int, paths: List[str], entries: List[Entry], loggingOutcome: str + } + pure def unrendered(code: int): bool = code == VALUE_UNRENDERABLE + // A value field: left out with `value` off, `null` for a side with no rendering. + pure def valueField(logging: Logging, code: int): int = + if (not(logging.value)) OMITTED else if (unrendered(code)) NO_RENDERING else code + // The warning a confirmed mismatch logs under the job's captured leaves, + // over two scalar values (W10's root cases; the walk over nested values is + // the protocol vectors'). Each side is rendered once when `value` is on, or + // when `diff` is and no hook replaces the walk; a rendered side with no + // rendering is a `render_error`, its value field null and the walk not run. + // The hook runs only when `diff` is on; the model's hook throws, a + // `hook_error` that nulls every diff field it carries and never falls back + // to the walk (D6), ranking above `render_error`. The walk reports nothing + // for identical renderings, else one entry at the root with both sides. + pure def warningOf(job: Job, cached: int, result: int, ageMs: int): MismatchWarning = { + val diffOn = job.logging.diff != DIFF_OFF + val rendered = job.logging.value or (diffOn and not(job.diffHook)) + val renderFails = rendered and (unrendered(cached) or unrendered(result)) + val hookFails = diffOn and job.diffHook + val listed = diffOn and not(hookFails) and not(renderFails) + val found = if (cached == result) List() else List({ path: "", cached: cached, source: result }) + { + ageMs: ageMs, cachedValue: valueField(job.logging, cached), sourceValue: valueField(job.logging, result), + diff: job.logging.diff, diffNull: diffOn and not(listed), + count: if (listed) found.length() else 0, + paths: if (listed) found.foldl(List(), (paths, entry) => paths.append(entry.path)) else List(), + entries: if (listed and job.logging.diff == DIFF_VALUES) found else List(), + loggingOutcome: if (hookFails) "hook_error" else if (renderFails) "render_error" else "" + } + } + // The one `shadowMismatchLogging` outcome of a warning attempt (D11, D15): + // the construction failure the warning names, else `logger_error` when the + // logger failed after receiving it, else `logged`. + pure def loggingOutcomeOf(warning: MismatchWarning, loggerFailed: bool): str = + if (warning.loggingOutcome != "") warning.loggingOutcome else if (loggerFailed) "logger_error" else "logged" + pure def recordMismatchWarning(state: DiagnosedShadowed[r, q], warning: MismatchWarning, + outcome: str): DiagnosedShadowed[r, q] = { + val counted = Diagnostics::recordWarning(state) + { + d: { + mismatchWarnings: counted.d.mismatchWarnings.append(warning), + loggingOutcomes: counted.d.loggingOutcomes.append(outcome), ...counted.d + }, + ...counted + } + } // The verdict the drivers observe, recorded beside the label the settlement // reported for the job: a `match` or a confirmed `mismatch` samples the C0 // frame's age at the verdict, clamped to zero after a wall rollback (C57), - // and a confirmed mismatch warns under the logging policy the job captured - // (C60). It warns exactly when the admission's mismatch-logging policy was - // on (`Shadowing.logging`). A timed-out job reports no label and so no verdict. - pure def recordVerdict(before: DiagnosedShadowed[r, q], after: DiagnosedShadowed[r, q], job: Job): DiagnosedShadowed[r, q] = { + // and a confirmed mismatch warns once when the leaves the job captured at + // admission opt in (`logs`), its content judged by `warningOf` and its one + // logging outcome by `loggingOutcomeOf` under the logger's failure input. + // A timed-out job reports no label and so no verdict. + pure def recordVerdict(before: DiagnosedShadowed[r, q], after: DiagnosedShadowed[r, q], job: Job, + loggerFailed: bool): DiagnosedShadowed[r, q] = { val labels = after.o.shadow val outcome = if (labels.length() > before.o.shadow.length()) labels.nth(labels.length() - 1) else "" if (outcome != "match" and outcome != "mismatch") after else { - val age = Clock::wallOf(after) - job.stamp - val warned = if (outcome == "mismatch" and job.logging) Diagnostics::recordWarning(after) else after - Diagnostics::recordAge(warned, if (age < 0) 0 else age) + val elapsed = Clock::wallOf(after) - job.stamp + val age = if (elapsed < 0) 0 else elapsed + val warning = warningOf(job, Payloads::valueOf(job.cached), sourceOf(after, job).result, age) + val warned = if (outcome == "mismatch" and logs(job.logging)) + recordMismatchWarning(after, warning, loggingOutcomeOf(warning, loggerFailed)) else after + Diagnostics::recordAge(warned, age) } } - // The drivers release a job's held decode: the settlement, its verdict recorded. + // The drivers release a job's held decode: the settlement, its verdict + // recorded. A decode never confirms a mismatch, so no logger runs here. pure def releaseLoad(state: DiagnosedShadowed[r, q], layout: Serving::Layout, load: int, failed: bool, comparison: Comparison, budget: int): DiagnosedShadowed[r, q] = - recordVerdict(state, settleJobLoad(state, layout, load, failed, comparison, budget), decoderOf(state, load)) - // The drivers release a job's held read: a visible frame stamped after the - // wall clock reports its offset on the shadow layer whatever follows, then - // the settlement, its verdict recorded. - pure def releaseRead(state: DiagnosedShadowed[r, q], layout: Serving::Layout, read: int, failed: bool): DiagnosedShadowed[r, q] = { + recordVerdict(state, settleJobLoad(state, layout, load, failed, comparison, budget), decoderOf(state, load), false) + // The drivers release a job's held read, the logger failing or not: a + // visible frame stamped after the wall clock reports its offset on the + // shadow layer whatever follows, then the settlement, its verdict recorded. + pure def releaseReadWithLogger(state: DiagnosedShadowed[r, q], layout: Serving::Layout, read: int, failed: bool, + loggerFailed: bool): DiagnosedShadowed[r, q] = { val job = readerOf(state, read) val key = sourceOf(state, job).key val stamp = state.created.nth(key) val offset = stamp - Clock::wallOf(state) val measured = if (offset > 0 and Remote::observe(state, key, layout.operationsPerEntity, 0, failed).visible) Diagnostics::recordFutureOffset(state, Diagnostics::REMOTE_SHADOW_LAYER, offset) else state - recordVerdict(measured, settleJobRead(measured, layout, read, failed), job) + recordVerdict(measured, settleJobRead(measured, layout, read, failed), job, loggerFailed) } + // A profile whose logger never fails. + pure def releaseRead(state: DiagnosedShadowed[r, q], layout: Serving::Layout, read: int, + failed: bool): DiagnosedShadowed[r, q] = + releaseReadWithLogger(state, layout, read, failed, false) + // Opt-in record for a profile's own properties, never read by the + // traversal: what each verdict saw, in verdict order (parallel to `d.ages`): + // its label, the leaves and diff hook its job captured, the C0's decoded + // value, the source's result and whether the logger failed. + type Verdict = { outcome: str, logging: Logging, diffHook: bool, cached: int, result: int, loggerFailed: bool } + pure def recordVerdictInputs(before: DiagnosedShadowed[{ verdicts: List[Verdict] | r }, q], + after: DiagnosedShadowed[{ verdicts: List[Verdict] | r }, q], job: Job, + loggerFailed: bool): DiagnosedShadowed[{ verdicts: List[Verdict] | r }, q] = + if (after.d.ages.length() == before.d.ages.length()) after + else { + verdicts: after.verdicts.append({ + outcome: after.o.shadow.nth(after.o.shadow.length() - 1), logging: job.logging, diffHook: job.diffHook, + cached: Payloads::valueOf(job.cached), result: sourceOf(before, job).result, loggerFailed: loggerFailed + }), + ...after + } // Delivery from inside the kernel, by kind: a job budget (nothing for a job // a source deadline of the same round already ended), a read deadline, or a // source deadline through the diagnosed expiry, then the job waiting on the diff --git a/formal/replay/features.d.mts b/formal/replay/features.d.mts index 49de0a64..e02ae753 100644 --- a/formal/replay/features.d.mts +++ b/formal/replay/features.d.mts @@ -2,7 +2,9 @@ import type { Observation, Input } from "../../typescript/test/formal/behavior-d import type { Profile } from "../../typescript/test/formal/feature-profile.js"; export type Projected = Omit & { calls: number[] }; export interface Inspection { instance: number; activeLeaders: number; activeFollowers: number; oldestLeaderAgeMs: number | null } -export interface Diagnostics { inspections?: Inspection[]; warnings: number; ages: number[]; coalesced: string[]; fallbackErrors: string[]; configErrors?: number; futureOffsets?: Array<{ layer: string; offsetMs: number }> } +export interface MismatchEntry { path: string; cached: number; source: number } +export interface MismatchWarning { ageSeconds: number; cachedValue: number; sourceValue: number; diff: "off" | "paths" | "values"; diffNull: boolean; count: number; paths: string[]; entries: MismatchEntry[]; loggingOutcome: "" | "hook_error" | "render_error" } +export interface Diagnostics { inspections?: Inspection[]; warnings: number; ages: number[]; coalesced: string[]; fallbackErrors: string[]; configErrors?: number; futureOffsets?: Array<{ layer: string; offsetMs: number }>; mismatchWarnings?: MismatchWarning[]; loggingOutcomes?: string[] } export interface ReadIO { budgets: number[]; aborted: number[]; sourceErrors: number[] } export interface Marker { cutoffMs: number; ttlMs: number } export interface Step { policyErrors?: Array<{ layer: string; errorType: string }>; compression?: string[]; markers?: Marker[]; io?: ReadIO; action: string; choice: number; expected: Projected; diagnostics?: Diagnostics } diff --git a/formal/replay/features.mjs b/formal/replay/features.mjs index 3a4e3c46..8aa352d3 100644 --- a/formal/replay/features.mjs +++ b/formal/replay/features.mjs @@ -10,10 +10,7 @@ import { shadowLayersProfile } from "./profiles/shadow-layers.mjs"; import { sourceBudgetsProfile } from "./profiles/source-budgets.mjs"; import { shadowReadDeadlinesProfile } from "./profiles/shadow-read-deadlines.mjs"; import { darkLayersProfile } from "./profiles/dark-layers.mjs"; -const settle = (op) => ({ - ...(op === "resolve" ? { choices: [1, 2] } : {}), - input: (choice, o) => op === "resolve" ? { op, loader: o.loaders - 1, value: choice } : { op, loader: o.loaders - 1 }, -}); +const rejectLatest = { input: (_, o) => ({ op: "reject", loader: o.loaders - 1 }) }; // Portable success codes reserve 0/3/4 for pending/source-error/deadline. export const successValues = [1, 2, undefined, null, false, 0, ""]; const resolveValue = (maxSources) => ({ @@ -43,6 +40,28 @@ const policyOverlays = overlays.concat(overlays.map((overlay) => ({ ...overlay, ]); const layerPolicies = [{}, { requestLocal: false }, { ramp: { local: 0 } }, { ramp: { remote: 0 } }, { ramp: { local: 0, remote: 0 } }, { requestLocal: false, ramp: { local: 0, remote: 0 } }]; +// The shadow profile's default logging inputs by init fixture, and its +// runtime logging overlays by logPolicy choice (choices 6 and 7 are the first +// two rows of the shorthand table, choices 11 and 12 write one leaf off; +// "invalid" has the wrong type for any leaf). +const shadowDefaultLogging = (choice) => (choice >= 4 && choice < 8) || choice === 13 ? { logMismatches: true } + : choice === 14 ? { mismatchLogging: { diff: "paths" } } : choice === 15 ? { mismatchLogging: { value: false } } : {}; +const shadowLogOverlays = [ + { logMismatches: false }, { logMismatches: true }, { logMismatches: "invalid" }, + { mismatchLogging: { diff: "paths" } }, { mismatchLogging: { diff: "values" } }, + { logMismatches: false, mismatchLogging: { value: true } }, { logMismatches: true, mismatchLogging: { value: false } }, + { logMismatches: false, mismatchLogging: { diff: "paths" } }, { mismatchLogging: { value: true, diff: "invalid" } }, + { mismatchLogging: { value: "invalid", diff: "paths" } }, { logMismatches: "invalid", mismatchLogging: { diff: "values" } }, + { mismatchLogging: { diff: "off" } }, { mismatchLogging: { value: false } }, +]; +// Choice 12 settles the source with the driver's value that has no JSON +// rendering (D13); the caller observes it as { unrenderable: true }. +const UNRENDERABLE = 12; +const shadowResolve = { + choices: [1, 2, UNRENDERABLE], + input: (choice, o) => choice === UNRENDERABLE ? { op: "resolve", loader: o.loaders - 1, unrenderable: true } + : { op: "resolve", loader: o.loaders - 1, value: choice }, +}; const shadowSeed = { choices: [1, 2, 3, 4, 5, 6, 7, 8], input: (choice) => choice <= 2 ? { op: "seed", value: choice } : choice === 7 ? { op: "seed", payloadText: JSON.stringify("café") } : choice === 8 ? { op: "seed", payloadHex: "22636166c3a922" } : choice === 6 ? { op: "seed", payloadText: " 1" } @@ -178,26 +197,27 @@ export const profiles = { }, }, shadow: { - explicitInputs: true, diagnosticFutureOffsets: true, - diagnosticAge: "shadowAge", diagnosticConfigErrors: true, initChoices: Array.from({ length: 13 }, (_, i) => i), + explicitInputs: true, diagnosticFutureOffsets: true, diagnosticWarnings: true, + diagnosticAge: "shadowAge", diagnosticConfigErrors: true, initChoices: Array.from({ length: 16 }, (_, i) => i), fixture: (choice) => ({ policy: { ttlSec: { remote: 60 }, ramp: { remote: 0 }, - shadow: { ramp: 100, ...(choice < 4 || choice >= 8 ? {} : { logMismatches: true }) } }, tracked: true, shadowHook: choice !== 8, + shadow: { ramp: 100, ...shadowDefaultLogging(choice) } }, tracked: true, shadowHook: choice !== 8, ...(choice % 4 === 0 || choice >= 11 ? {} : { comparator: ["equal", "unequal", "error"][choice === 10 ? 2 : choice % 4 - 1] }), ...(choice === 9 || choice === 10 ? { comparisonMs: 10 } : {}), - ...(choice >= 11 ? { sourceWorkMs: choice === 11 ? 9 : 10 } : {}), - observe: ["shadowAge", "mismatchWarning", "coalesced", "error", "futureOffset"] }), + ...(choice === 11 || choice === 12 ? { sourceWorkMs: choice === 11 ? 9 : 10 } : {}), + ...(choice === 13 ? { mismatchDiff: "error" } : {}), + observe: ["shadowAge", "mismatchWarning", "shadowMismatchLogging", "coalesced", "error", "futureOffset"] }), setup: [{ op: "faults", value: { holdReads: true, holdLoads: true, holdDumps: true, holdWrites: true } }], actions: { - beginCall: { input: () => ({ op: "begin" }) }, resolveLoader: settle("resolve"), rejectLoader: settle("reject"), + beginCall: { input: () => ({ op: "begin" }) }, resolveLoader: shadowResolve, rejectLoader: rejectLatest, releaseRead: release("read"), releaseLoad: release("load"), releaseDump: release("dump"), releaseWrite: release("write"), advance: advance([1, 10]), seed: shadowSeed, reencode: shadowSeed, seedUnicode: { ...shadowSeed, choices: [7, 8] }, invalidate: { choices: [0, 20], input: (choice) => ({ op: "invalidate", futureBufferMs: choice }) }, - readFault: fault("read"), loadFault: fault("load"), dumpFault: fault("dump"), writeFault: fault("write"), + readFault: fault("read"), loadFault: fault("load"), dumpFault: fault("dump"), writeFault: fault("write"), loggerFault: fault("logger"), rollbackWall: { input: () => ({ op: "shiftWall", ms: -1000 }) }, advanceWall: { choices: [1, 60000], input: choice => ({ op: "shiftWall", ms: choice }) }, shadowPolicy: { choices: [0, 1, 2], input: (choice) => ({ op: "policy", value: { shadow: { ramp: [100, 0, 101][choice] } } }) }, - logPolicy: { choices: [0, 1, 2], input: (choice) => ({ op: "policy", value: { shadow: { logMismatches: choice === 2 ? "invalid" : choice === 1 } } }) }, + logPolicy: { choices: shadowLogOverlays.map((_, i) => i), input: (choice) => ({ op: "policy", value: { shadow: shadowLogOverlays[choice] } }) }, }, }, }; @@ -215,7 +235,7 @@ function observation(raw, context) { return [key, item.map((entry) => { if (key === "calls" || key === "writeTtls") { const integer = itfInteger(entry, context); - if (key === "calls" && (integer < 0 || integer > 9 && integer !== 11)) + if (key === "calls" && (integer < 0 || integer > 9 && integer !== 11 && integer !== UNRENDERABLE)) throw new Error(`${context}: unsupported caller outcome`); return integer; } @@ -226,10 +246,17 @@ function observation(raw, context) { })); return result; } -function diagnostics(raw, context, configErrors = false, futureOffsets = false, inspect = false) { +// The warning content lists a profile asserts only with diagnosticWarnings. A +// profile whose drivers never log a mismatch may carry them, empty, or not at +// all (a trace generated before they existed), so the differential compares +// it across both texts. +const warningContentFields = ["loggingOutcomes", "mismatchWarnings"]; +function diagnostics(raw, context, configErrors = false, futureOffsets = false, inspect = false, warningContent = false) { const value = record(raw, context); - const fields = ["ages", "coalesced", "fallbackErrors", "warnings", ...(configErrors ? ["configErrors"] : []), ...(futureOffsets ? ["futureOffsets"] : []), ...(inspect ? ["inspections"] : [])]; - if (Object.keys(value).sort().join() !== fields.sort().join() || !Array.isArray(value.ages)) + const fields = ["ages", "coalesced", "fallbackErrors", "warnings", ...(configErrors ? ["configErrors"] : []), ...(futureOffsets ? ["futureOffsets"] : []), + ...(inspect ? ["inspections"] : []), ...(warningContent ? warningContentFields : [])]; + const carried = Object.keys(value).filter(key => warningContent || !warningContentFields.includes(key) || !Array.isArray(value[key]) || value[key].length > 0); + if (carried.sort().join() !== fields.sort().join() || !Array.isArray(value.ages)) throw new Error(`${context}: invalid diagnostics`); const labels = (items, allowed) => { if (!Array.isArray(items) || items.some(item => typeof item !== "string" || !allowed.includes(item))) @@ -260,9 +287,36 @@ function diagnostics(raw, context, configErrors = false, futureOffsets = false, activeFollowers: itfInteger(entry.activeFollowers, context), oldestLeaderAgeMs: age === -1 ? null : age }; }); })() : []; - return { ...(inspect ? { inspections } : {}), ...(futureOffsets ? { futureOffsets: offsets } : {}), ...(configErrors ? { configErrors: itfInteger(value.configErrors, context) } : {}), warnings: itfInteger(value.warnings, context), ages: value.ages.map(age => itfInteger(age, context) / 1000), + return { ...(inspect ? { inspections } : {}), ...(futureOffsets ? { futureOffsets: offsets } : {}), ...(configErrors ? { configErrors: itfInteger(value.configErrors, context) } : {}), + ...(warningContent ? { mismatchWarnings: mismatchWarnings(value.mismatchWarnings, context), loggingOutcomes: labels(value.loggingOutcomes, loggingOutcomes) } : {}), + warnings: itfInteger(value.warnings, context), ages: value.ages.map(age => itfInteger(age, context) / 1000), coalesced: labels(value.coalesced, ["process", "request_local"]), fallbackErrors: labels(value.fallbackErrors, ["noop", "local", "remote", "request_local"]) }; } +// One confirmed-mismatch warning's content as the shadow profile predicts it +// (D12), in the record the projection builds from a driver's event: the age in +// seconds, each value field and entry side as a value code (-1 left out, 0 +// null), the diff level whose fields it carries and whether they are null, the +// uncut count, the paths, the entries (at `values`) and the failure it names. +const loggingOutcomes = ["logged", "hook_error", "render_error", "logger_error"]; +const warningFields = "ageMs,cachedValue,count,diff,diffNull,entries,loggingOutcome,paths,sourceValue"; +function mismatchWarnings(raw, context) { + if (!Array.isArray(raw)) throw new Error(`${context}: missing mismatch warnings`); + return raw.map(item => { + const warning = record(item, context); + if (Object.keys(warning).sort().join() !== warningFields || !Array.isArray(warning.paths) || !Array.isArray(warning.entries) + || !["off", "paths", "values"].includes(warning.diff) || typeof warning.diffNull !== "boolean" + || !["", "hook_error", "render_error"].includes(warning.loggingOutcome) || warning.paths.some(path => typeof path !== "string")) + throw new Error(`${context}: invalid mismatch warning`); + return { ageSeconds: itfInteger(warning.ageMs, context) / 1000, cachedValue: itfSignedInteger(warning.cachedValue, context), + sourceValue: itfSignedInteger(warning.sourceValue, context), diff: warning.diff, diffNull: warning.diffNull, + count: itfInteger(warning.count, context), paths: warning.paths, entries: warning.entries.map(raw => { + const entry = record(raw, context); + if (Object.keys(entry).sort().join() !== "cached,path,source" || typeof entry.path !== "string") + throw new Error(`${context}: invalid mismatch entry`); + return { path: entry.path, cached: itfSignedInteger(entry.cached, context), source: itfSignedInteger(entry.source, context) }; + }), loggingOutcome: warning.loggingOutcome }; + }); +} function readIO(raw, context, callCount) { const value = record(raw, context); if (Object.keys(value).sort().join() !== "aborted,budgets,sourceErrors" || !Array.isArray(value.budgets) || !Array.isArray(value.aborted) || !Array.isArray(value.sourceErrors)) @@ -315,7 +369,8 @@ export function parseTrace(raw, path, profile) { ...(profile.compressionIO ? { compression: compressionIO(record(state.s, context).compression, context) } : {}), ...(profile.markerIO ? { markers: markerIO(record(state.s, context).markers, context) } : {}), ...(profile.readIO ? { io: readIO(record(state.s, context).io, context, expected.calls.length) } : {}), - ...(profile.diagnosticAge === undefined ? {} : { diagnostics: diagnostics(record(state.s, context).d, context, profile.diagnosticConfigErrors, profile.diagnosticFutureOffsets, profile.diagnosticInspections) }) }; + ...(profile.diagnosticAge === undefined ? {} : { diagnostics: diagnostics(record(state.s, context).d, context, profile.diagnosticConfigErrors, + profile.diagnosticFutureOffsets, profile.diagnosticInspections, profile.diagnosticWarnings) }) }; }) }; } function valueCode(value) { @@ -334,8 +389,53 @@ function valueCode(value) { return 11; if (typeof v === "object" && v.absent === true) return 5; + if (typeof v === "object" && v.unrenderable === true) + return UNRENDERABLE; return 10; } +// A logged JSON value as the shadow profile's value code: its decoded values +// (`1`, `2`, `"café"` as 3) and the portable scalars. +function jsonCode(value) { + const codes = [[1, 1], [2, 2], ["café", 3], [null, 6], [false, 7], [0, 8], ["", 9]]; + const found = codes.find(([json]) => json === value); + assertObservedDomain(found !== undefined, { value }, { value: "a value the shadow profile logs" }); + return found[1]; +} +function parsedField(text, field) { + if (typeof text !== "string") throw new Error(`Invalid actual ${field} shape`); + try { return JSON.parse(text); } + catch { return assertObservedDomain(false, { [field]: text }, { [field]: "JSON text" }); } +} +// A value field or an entry side: -1 when left out, 0 when logged as null. +const sideCode = (holder, side, parse) => !Object.hasOwn(holder, side) ? -1 + : holder[side] === null && parse ? 0 : jsonCode(parse ? parsedField(holder[side], side) : holder[side]); +// A driver's mismatchWarning event as the shadow profile's warning record: the +// diff level is the most detailed diff field it carries; the paths, the count +// and diffJson are null together or not at all, and diffJson's entries carry +// the same paths as mismatchPathsJson (the comparison checks both). +function projectWarning(event) { + const has = field => Object.hasOwn(event, field); + if (typeof event.cachedValueAgeSeconds !== "number") + throw new Error("Missing actual warning age"); + const diff = has("diffJson") ? "values" : has("mismatchPathsJson") ? "paths" : "off"; + const diffNull = has("mismatchPathsJson") && event.mismatchPathsJson === null; + assertObservedDomain(has("mismatchPathsJson") === has("mismatchPathCount") && (diff !== "values" || has("mismatchPathsJson")) + && (diff === "off" || ((event.mismatchPathCount === null) === diffNull && (diff !== "values" || (event.diffJson === null) === diffNull))), + { mismatchPathsJson: event.mismatchPathsJson, mismatchPathCount: event.mismatchPathCount, diffJson: event.diffJson }, + { diffFields: "carried together and null together" }); + const listed = diff !== "off" && !diffNull; + const paths = listed ? parsedField(event.mismatchPathsJson, "mismatchPathsJson") : []; + const entries = listed && diff === "values" ? parsedField(event.diffJson, "diffJson") : []; + assertObservedDomain(Array.isArray(paths) && paths.every(path => typeof path === "string") && Array.isArray(entries) + && entries.every(entry => entry !== null && typeof entry === "object" && typeof entry.path === "string" + && Object.keys(entry).every(key => ["path", "cached", "source"].includes(key))) + && (!listed || Number.isSafeInteger(event.mismatchPathCount)), + { mismatchPathsJson: event.mismatchPathsJson, diffJson: event.diffJson }, { diffFields: "paths, entries and an integer count" }); + return { ageSeconds: event.cachedValueAgeSeconds, cachedValue: sideCode(event, "cachedValueJson", true), sourceValue: sideCode(event, "sourceValueJson", true), + diff, diffNull, count: listed ? event.mismatchPathCount : 0, paths, + entries: entries.map(entry => ({ path: entry.path, cached: sideCode(entry, "cached", false), source: sideCode(entry, "source", false) })), + loggingOutcome: event.loggingOutcome ?? "" }; +} function project(o) { return { ...o, calls: o.calls.map((c) => c.status === "pending" ? 0 : c.status === "value" ? valueCode(c) @@ -443,7 +543,7 @@ function projectBaseObservation(profile, observed) { const ages = []; const futureOffsets = []; let warnings = 0, configErrors = 0; - const coalesced = [], fallbackErrors = [], inspections = []; + const coalesced = [], fallbackErrors = [], inspections = [], mismatchWarnings = [], loggingOutcomes = []; const outcomes = profile.diagnosticAge === "shadowAge" ? observed.shadow.filter(x => x === "match" || x === "mismatch") : observed.recovery.filter(x => x === "served"); for (const event of events) { @@ -495,6 +595,12 @@ function projectBaseObservation(profile, observed) { else if (event.event === "mismatchWarning") { assert.deepEqual(event.outcome, "mismatch"); warnings++; + if (profile.diagnosticWarnings) mismatchWarnings.push(projectWarning(event)); + } + else if (event.event === "shadowMismatchLogging" && profile.diagnosticWarnings) { + if (typeof event.outcome !== "string") + throw new Error("Invalid actual mismatch logging outcome shape"); + loggingOutcomes.push(event.outcome); } else { assert.deepEqual(event.event, profile.diagnosticAge); @@ -504,7 +610,8 @@ function projectBaseObservation(profile, observed) { ages.push(event.seconds); } } - return { o: project(base), d: { warnings, ages, coalesced, fallbackErrors, ...(profile.diagnosticInspections ? { inspections } : {}), ...(profile.diagnosticFutureOffsets ? { futureOffsets } : {}), ...(profile.diagnosticConfigErrors ? { configErrors } : {}) } }; + return { o: project(base), d: { warnings, ages, coalesced, fallbackErrors, ...(profile.diagnosticInspections ? { inspections } : {}), ...(profile.diagnosticFutureOffsets ? { futureOffsets } : {}), ...(profile.diagnosticConfigErrors ? { configErrors } : {}), + ...(profile.diagnosticWarnings ? { mismatchWarnings, loggingOutcomes } : {}) } }; } export function expectedObservation(step) { return { o: step.expected, ...(step.policyErrors === undefined ? {} : { policyErrors: step.policyErrors }), ...(step.diagnostics === undefined ? {} : { d: step.diagnostics }), ...(step.io === undefined ? {} : { io: step.io }), ...(step.markers === undefined ? {} : { markers: step.markers }), ...(step.compression === undefined ? {} : { compression: step.compression }) }; } export function featureInput(profile, action, choice, observed, environment) { const binding = profile.actions[action]; if (!binding || (binding.choices ? !binding.choices.includes(choice) : choice !== -1 && choice !== 0)) diff --git a/formal/replay/protocol.schema.json b/formal/replay/protocol.schema.json index 59db6884..cbd9dfe1 100644 --- a/formal/replay/protocol.schema.json +++ b/formal/replay/protocol.schema.json @@ -455,6 +455,29 @@ ], "additionalProperties": false }, + { + "type": "object", + "description": "Resolve the loader with the driver's native value that has no JSON rendering (D13): a throwing toJSON, a failing MarshalJSON or Serialize, a non-serializable object. The caller observes it as { unrenderable: true }.", + "properties": { + "op": { + "const": "resolve" + }, + "loader": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "unrenderable": { + "const": true + } + }, + "required": [ + "op", + "loader", + "unrenderable" + ], + "additionalProperties": false + }, { "type": "object", "properties": { @@ -662,6 +685,10 @@ "observer": { "type": "boolean" }, + "logger": { + "type": "boolean", + "description": "The logger callback throws after the driver records what it was given; metric observers keep working." + }, "holdReads": { "type": "boolean" }, @@ -1005,6 +1032,19 @@ ], "additionalProperties": false }, + "unrenderable": { + "type": "object", + "description": "A call that completed with the driver's value that has no JSON rendering (the resolve command's unrenderable result), recognized by identity, never through JSON.", + "properties": { + "unrenderable": { + "const": true + } + }, + "required": [ + "unrenderable" + ], + "additionalProperties": false + }, "callResult": { "description": "Outcome of one begun getOrLoad call, in begin order. Error identities are driver-local indices, never messages: source:N is the N-th controlled source failure, timeout:N the N-th fallback deadline error, unexpected: anything else.", "oneOf": [ @@ -1038,6 +1078,9 @@ }, { "$ref": "#/$defs/absent" + }, + { + "$ref": "#/$defs/unrenderable" } ] } @@ -1643,7 +1686,7 @@ }, { "type": "object", - "description": "Details object of the shadow validation mismatch warning log. Value previews are present only when the policy enables mismatch logging with details.", + "description": "Details object of the shadow validation mismatch warning log, each member recorded exactly when the warning carries it: the value fields with mismatchLogging.value, mismatchPathsJson and mismatchPathCount at diff paths or values, diffJson at values, loggingOutcome when construction failed.", "properties": { "event": { "const": "mismatchWarning" @@ -1674,6 +1717,77 @@ "string", "null" ] + }, + "cachedValueAgeSeconds": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "description": "The age the shadowAge metric records for the same verdict, in seconds." + }, + "mismatchPathsJson": { + "type": [ + "string", + "null" + ], + "description": "The warning's mismatchPaths array as one JSON string, null when the field is null." + }, + "mismatchPathCount": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 9007199254740991 + }, + "diffJson": { + "type": [ + "string", + "null" + ] + }, + "loggingOutcome": { + "enum": [ + "hook_error", + "render_error" + ] + } + }, + "required": [ + "event", + "cacheNamespace", + "useCase", + "keyType", + "outcome" + ], + "additionalProperties": { + "$ref": "#/$defs/primitive" + } + }, + { + "type": "object", + "description": "One shadowMismatchLogging metric event per confirmed-mismatch warning attempt (D11).", + "properties": { + "event": { + "const": "shadowMismatchLogging" + }, + "cacheNamespace": { + "type": "string" + }, + "useCase": { + "type": "string" + }, + "keyType": { + "type": "string" + }, + "outcome": { + "enum": [ + "logged", + "hook_error", + "render_error", + "logger_error" + ] } }, "required": [ @@ -2246,6 +2360,12 @@ "type": "boolean", "description": "false removes the shadowValidation metrics hook. Default true." }, + "mismatchDiff": { + "enum": [ + "error" + ], + "description": "Per-call shadowMismatchDiff hook: error throws (or returns an error, per the port). Omitted: no hook, the built-in walk runs." + }, "observerFailure": { "type": "boolean", "description": "Every observer and logger callback throws. Default false." @@ -2280,6 +2400,7 @@ "fallback", "serialization", "mismatchWarning", + "shadowMismatchLogging", "writeDispatch", "marker", "coalescingState" diff --git a/formal/replay/witnesses/shadow-diagnostics.mjs b/formal/replay/witnesses/shadow-diagnostics.mjs index 389c44be..3adb0bcc 100644 --- a/formal/replay/witnesses/shadow-diagnostics.mjs +++ b/formal/replay/witnesses/shadow-diagnostics.mjs @@ -6,6 +6,15 @@ const resolve = command("resolveLoader", 2), load = command("releaseLoad"); const rollback = command("rollbackWall"); const mismatch = { calls: [2], reads: 2, loads: 1, writes: 0, comparisons: 1, shadow: ["mismatch"] }; const futureShadow = [{ layer: "remote_shadow", offsetMs: 1000 }]; +// A confirmed mismatch at C0 age 0 (the regressions' confirmMismatch) and the +// warnings it logs: neither value field nor diff field to start from. +const confirm = value => [seed, begin, read, command("resolveLoader", value), load, read]; +const logPolicy = choice => command("logPolicy", choice); +const omitted = { ageMs: 0, cachedValue: -1, sourceValue: -1, diff: "off", diffNull: false, count: 0, paths: [], entries: [], loggingOutcome: "" }; +const everything = { ...omitted, cachedValue: 1, sourceValue: 2, diff: "values", count: 1, paths: [""], entries: [{ path: "", cached: 1, source: 2 }] }; +const failedDiff = (sourceValue, loggingOutcome) => ({ ...omitted, cachedValue: 1, sourceValue, diff: "values", diffNull: true, loggingOutcome }); +const warned = (step, warning, outcome, more = {}) => check(step, { shadow: ["mismatch"] }, + { mismatchWarnings: [warning], loggingOutcomes: [outcome], ...more }); export const shadowDiagnosticsWitnessRules = [ rule("omitted-shadow-logging-keeps-mismatch-metrics-only", "omittedLoggingKeepsMismatchMetricsOnlyTest", @@ -32,6 +41,31 @@ export const shadowDiagnosticsWitnessRules = [ check(4, { calls: [4], shadow: ["timeout"], reads: 1 }, { futureOffsets: [] }), check(5, { calls: [4], shadow: ["timeout"], reads: 1, loads: 0, dumps: 0, writes: 0 }, { futureOffsets: [{ layer: "remote_shadow", offsetMs: 990 }] })), + // The record's shorthand table, one rule per row. + rule("shorthand-true-with-value-false-logs-diff-only", "trueShorthandWithValueFalseLogsDiffOnlyTest", + [init(0), logPolicy(6), ...confirm(2)], warned(7, { ...everything, cachedValue: -1, sourceValue: -1 }, "logged", { configErrors: 0 })), + rule("shorthand-false-with-diff-paths-logs-paths-only", "falseShorthandWithDiffPathsLogsPathsOnlyTest", + [init(0), logPolicy(7), ...confirm(2)], warned(7, { ...omitted, diff: "paths", count: 1, paths: [""] }, "logged", { configErrors: 0 })), + rule("runtime-false-shorthand-overrides-default-diff", "runtimeFalseShorthandOverridesDefaultDiffTest", + [init(14), logPolicy(0), ...confirm(2)], + check(7, { shadow: ["mismatch"] }, { warnings: 0, ages: [0], mismatchWarnings: [], loggingOutcomes: [], configErrors: 0 })), + rule("runtime-true-shorthand-overrides-default-value", "runtimeTrueShorthandOverridesDefaultValueTest", + [init(15), logPolicy(1), ...confirm(2)], warned(7, everything, "logged")), + // A leaf the runtime overlay writes off, over the logMismatches default. + rule("runtime-off-diff-overrides-default-diff", "runtimeOffDiffOverridesDefaultDiffTest", + [init(4), logPolicy(11), ...confirm(2)], warned(7, { ...omitted, cachedValue: 1, sourceValue: 2 }, "logged", { configErrors: 0 })), + rule("runtime-false-value-overrides-default-value", "runtimeFalseValueOverridesDefaultValueTest", + [init(4), logPolicy(12), ...confirm(2)], warned(7, { ...everything, cachedValue: -1, sourceValue: -1 }, "logged", { configErrors: 0 })), + // Each failure outcome, and all three together. + rule("hook-error-nulls-diff-fields", "hookErrorNullsDiffFieldsTest", + [init(13), ...confirm(2)], warned(6, failedDiff(2, "hook_error"), "hook_error")), + rule("render-error-nulls-unrenderable-source", "renderErrorNullsUnrenderableSourceTest", + [init(4), ...confirm(12)], check(6, { calls: [12], shadow: ["mismatch"] }, + { mismatchWarnings: [failedDiff(0, "render_error")], loggingOutcomes: ["render_error"] })), + rule("logger-error-still-delivers-warning", "loggerErrorStillDeliversWarningTest", + [init(4), command("loggerFault", 1), ...confirm(2)], warned(7, everything, "logger_error")), + rule("hook-error-outranks-both-failures", "hookErrorOutranksBothFailuresTest", + [init(13), command("loggerFault", 1), ...confirm(12)], warned(7, failedDiff(0, "hook_error"), "hook_error")), ]; export function shadowDiagnosticsWitnesses(histories, recorder) { diff --git a/formal/replay/witnesses/shadow.mjs b/formal/replay/witnesses/shadow.mjs index 9482c9c7..d6412671 100644 --- a/formal/replay/witnesses/shadow.mjs +++ b/formal/replay/witnesses/shadow.mjs @@ -5,7 +5,8 @@ import { fidelityBinding } from "./fidelity.mjs"; // The shadow profile's witnesses, decided from the recorded inputs and the // asserted public channels alone. What a rule needs beyond the observation // deltas is shadowed here: the fixture the init choice names (comparator, -// comparison time, source work, hook, default logging), both clocks, the +// comparison time, source work, outcome and diff hooks, default logging +// inputs), the runtime logging inputs and the logger's failure, both clocks, the // frame a seed or a fill stored with its stamp, the watermark an invalidation // raised, the fault switches, and the one dark job the drivers run at a time // (its phase, its acquired C0 with the frame's stamp and the fence it observed, @@ -16,8 +17,9 @@ import { fidelityBinding } from "./fidelity.mjs"; // still carries the model's private predictions the fidelity check compares // the shadow after every step with the composed layout // (dialcache-shadow-conformance.qnt over the kernel library): the frame, its -// stamp and the watermark, the two clocks, the fault switches, the one live -// job as a shadow::Job record and the read or decode it holds by ordinal. +// stamp and the watermark, the two clocks, the fault switches, the logging +// inputs and the diff hook, the one live job as a shadow::Job record (its +// captured leaves included) and the read or decode it holds by ordinal. const IDLE = 0, C0_READ = 1, SOURCE_WAIT = 2, SNAPSHOT_DECODE = 3, CONFIRMATION_READ = 4, FILL_SERIALIZE = 5, FILL_WRITE = 6, JOB_DONE = 7; const NOT_STARTED = 0, PENDING = 1, ACCEPTED = 2, FAILED = 3, TIMED_OUT = 4; @@ -28,6 +30,38 @@ const BUDGET = 10, FRESH = 60000; const payloadBytes = payload => ({ 3: 1, 4: 2, 6: 5, 8: 7 })[payload] ?? payload; const decoded = payload => payload >= 7 ? 3 : [3, 5, 6].includes(payload) ? 1 : payload === 4 ? 2 : payload; const binary = payload => [3, 4, 5, 8].includes(payload); +// The source result with no JSON rendering (D13), the resolveLoader choice 12. +const UNRENDERABLE = 12; + +// Mismatch logging inputs as written, one config layer each: the +// logMismatches shorthand, then the mismatchLogging value and diff leaves; +// "" leaves one unwritten and "invalid" has the wrong type. The defaults by +// init fixture and the runtime overlays by logPolicy choice (11 and 12 write +// one leaf off). +const written = (shorthand, value, diff) => ({ shorthand, value, diff }); +const NO_LOGGING = written("", "", ""); +const defaultLoggingOf = choice => (choice >= 4 && choice < 8) || choice === 13 ? written("true", "", "") + : choice === 14 ? written("", "", "paths") : choice === 15 ? written("", "false", "") : NO_LOGGING; +const OVERLAYS = [written("false", "", ""), written("true", "", ""), written("invalid", "", ""), written("", "", "paths"), + written("", "", "values"), written("false", "true", ""), written("true", "false", ""), written("false", "", "paths"), + written("", "true", "invalid"), written("", "invalid", "paths"), written("invalid", "", "values"), + written("", "", "off"), written("", "false", "")]; +const DIFF_OFF_OVERLAY = 11, VALUE_FALSE_OVERLAY = 12; +// The leaves a job captures (record D10), per leaf: the overlay's own leaf +// when written, else what its shorthand stands for, else the same from the +// defaults. `value` is on only for "true" and `diff` only for a level it +// names; one configuration error covers a malformed shorthand and any leaf +// of the wrong type. +const standsFor = (shorthand, leaf) => shorthand === "" ? "" : shorthand === "true" ? (leaf === "value" ? "true" : "values") + : shorthand === "false" ? (leaf === "value" ? "false" : "off") : "invalid"; +const layerLeaf = (layer, leaf) => layer[leaf] !== "" ? layer[leaf] : standsFor(layer.shorthand, leaf); +export function capturedLogging(defaults, overlay) { + const inForce = leaf => layerLeaf(overlay, leaf) !== "" ? layerLeaf(overlay, leaf) : layerLeaf(defaults, leaf); + const value = inForce("value"), diff = inForce("diff"); + return { logging: { value: value === "true", diff: ["paths", "values"].includes(diff) ? diff : "off" }, + invalid: !["", "true", "false"].includes(overlay.shorthand) || !["", "true", "false"].includes(value) || !["", "off", "paths", "values"].includes(diff) }; +} +const logsMismatch = logging => logging.value || logging.diff !== "off"; function fixtureOf(choice) { return { @@ -35,15 +69,17 @@ function fixtureOf(choice) { comparisonMs: choice === 9 || choice === 10 ? BUDGET : 0, sourceWorkMs: choice === 12 ? BUDGET : choice === 11 ? BUDGET - 1 : 0, hook: choice !== 8, - defaultLog: choice >= 4 && choice < 8, + diffHook: choice === 13, + defaultLogging: defaultLoggingOf(choice), }; } function initial(choice) { const fixture = fixtureOf(choice); return { ...fixture, now: 10000, wall: 100000, frame: 0, created: 0, watermark: 0, c0: 0, c0Created: 0, phase: IDLE, abandoned: false, deadline: 0, source: NOT_STARTED, sourcePending: false, sourceValue: 0, fence: 0, writeStamp: 0, - readFailed: false, loadFailed: false, dumpFailed: false, writeFailed: false, - shadowActive: fixture.hook, log: fixture.defaultLog, invalidLog: false, invalidShadow: false, acceptedLog: false, acceptedInvalidLog: false, + readFailed: false, loadFailed: false, dumpFailed: false, writeFailed: false, loggerFailed: false, + shadowActive: fixture.hook, overlayLogging: NO_LOGGING, overlayChoice: -1, invalidShadow: false, + accepted: { logging: { value: false, diff: "off" }, invalid: false }, acceptedOverlayChoice: -1, predicted: { outcome: undefined, reads: 0, loads: 0, dumps: 0, writes: 0 } }; } // The profile's rules over the shadow; `predicted` collects what the step must @@ -65,7 +101,8 @@ function apply(before, action, choice) { s.deadline = s.now + BUDGET; s.now += s.sourceWorkMs; s.wall += s.sourceWorkMs; s.phase = s.shadowActive && !jobExpired ? C0_READ : JOB_DONE; s.source = PENDING; s.sourcePending = true; s.sourceValue = 0; s.abandoned = jobExpired; - s.c0 = 0; s.c0Created = 0; s.fence = 0; s.acceptedLog = s.log; s.acceptedInvalidLog = s.invalidLog; + s.c0 = 0; s.c0Created = 0; s.fence = 0; + s.accepted = capturedLogging(s.defaultLogging, s.overlayLogging); s.acceptedOverlayChoice = s.overlayChoice; if (s.shadowActive && !jobExpired) s.predicted.reads++; if (jobExpired) s.predicted.outcome = "timeout"; break; @@ -127,8 +164,9 @@ function apply(before, action, choice) { case "invalidate": if (!s.writeFailed) s.watermark = Math.max(s.watermark, s.wall + choice); break; case "rollbackWall": s.wall -= 1000; break; case "advanceWall": s.wall += choice; break; - case "logPolicy": s.log = choice === 1; s.invalidLog = choice === 2; s.invalidShadow = false; s.shadowActive = s.hook; break; - case "shadowPolicy": s.shadowActive = s.hook && choice === 0; s.log = s.defaultLog; s.invalidShadow = choice === 2; s.invalidLog = false; break; + case "logPolicy": s.overlayLogging = OVERLAYS[choice]; s.overlayChoice = choice; s.invalidShadow = false; s.shadowActive = s.hook; break; + case "shadowPolicy": s.shadowActive = s.hook && choice === 0; s.overlayLogging = NO_LOGGING; s.overlayChoice = -1; s.invalidShadow = choice === 2; break; + case "loggerFault": s.loggerFailed = choice === 1; break; case "readFault": s.readFailed = choice === 1; break; case "loadFault": s.loadFailed = choice === 1; break; case "dumpFault": s.dumpFailed = choice === 1; break; @@ -137,6 +175,50 @@ function apply(before, action, choice) { } return s; } +// The confirmed-mismatch warning a step logged, judged from its public content +// (the d channel's warning and logging outcome) beside the inputs that +// explain it: the leaves the job captured, the overlay admission read and +// whether admission reported it invalid, the diff hook, the source's result +// and the logger's failure (C60, D11 to D13). +function warningLabels(recorder, step, previous, before, after, fixture, reported) { + const warnings = step.diagnostics.mismatchWarnings, outcomes = step.diagnostics.loggingOutcomes; + const accepted = after.accepted.logging, overlay = after.acceptedOverlayChoice; + if (warnings.length === previous.diagnostics.mismatchWarnings.length) { + if (fixture === 14 && overlay === 0 && !logsMismatch(accepted)) recorder.credit("shorthand-row:3"); + return; + } + const warning = warnings.at(-1), outcome = outcomes.at(-1); + const valueLogged = warning.cachedValue !== -1; + recorder.credit(`warning-fields:${[valueLogged ? "value" : "", warning.diff === "off" ? "" : warning.diff].filter(Boolean).join("+")}`); + recorder.credit(`logging-outcome:${outcome}`); + const listed = warning.diff !== "off" && !warning.diffNull; + if (listed && warning.count === 0 && warning.paths.length === 0) recorder.credit("identical-renderings-log-no-paths"); + if (listed && warning.count === 1 && warning.paths.length === 1 && warning.paths[0] === "") recorder.credit("differing-renderings-log-root-entry"); + const hookRan = before.diffHook && accepted.diff !== "off", unrenderable = before.sourceValue === UNRENDERABLE; + if (unrenderable && valueLogged && warning.sourceValue === 0) recorder.credit("unrenderable-source-logs-null"); + // Once the hook replaces the walk only the value fields need the source's + // rendering, so the precedence shows only where they were logged and the + // source's field is null. + if (hookRan && unrenderable && valueLogged && warning.sourceValue === 0 && outcome === "hook_error") recorder.credit("hook-error-over-render-error"); + if (hookRan && before.loggerFailed && outcome === "hook_error") recorder.credit("hook-error-over-logger-error"); + if (!hookRan && unrenderable && before.loggerFailed && outcome === "render_error") recorder.credit("render-error-over-logger-error"); + if (before.diffHook && accepted.diff === "off" && outcome !== "hook_error") recorder.credit("hook-skipped-at-diff-off"); + if (outcome === "logger_error" && warning.loggingOutcome === "" && !warning.diffNull) recorder.credit("logger-error-keeps-warning-content"); + if (overlay === 6 && !valueLogged && warning.diff === "values") recorder.credit("shorthand-row:1"); + if (overlay === 7 && !valueLogged && warning.diff === "paths") recorder.credit("shorthand-row:2"); + if (fixture === 15 && overlay === 1 && valueLogged && warning.diff === "values") recorder.credit("shorthand-row:4"); + // A leaf the overlay writes off replaces its own inherited leaf, which the + // defaults alone turn on, and leaves the other inherited leaf on. + const inherited = capturedLogging(defaultLoggingOf(fixture), NO_LOGGING).logging; + if (overlay === DIFF_OFF_OVERLAY && inherited.diff !== "off" && inherited.value && valueLogged && warning.diff === "off") { + recorder.credit("written-off-leaf:diff"); + } + if (overlay === VALUE_FALSE_OVERLAY && inherited.value && inherited.diff !== "off" && !valueLogged && warning.diff === inherited.diff) { + recorder.credit("written-off-leaf:value"); + } + if (reported && (overlay === 8 || overlay === 9) && valueLogged !== (warning.diff !== "off")) recorder.credit("invalid-leaf-keeps-valid-leaf"); + if (reported && overlay === 10 && !valueLogged && warning.diff === "values") recorder.credit("invalid-shorthand-keeps-written-leaf"); +} // The shadow bound to the history: what the step predicts must be what the // drivers observed, or the history is refused rather than classified. function bind(path, index, s, previous, actual) { @@ -164,15 +246,18 @@ function modelView(shadow, o) { const source = o.calls.length - 1; const job = live ? { source, dark: true, cached: codeOf[shadow.c0], stamp: shadow.c0Created, fence: shadow.fence, - ttls: { localMs: 0, freshMs: FRESH, retentionMs: FRESH }, phase: phaseOf[shadow.phase], timedOut: shadow.abandoned, logging: shadow.acceptedLog, + ttls: { localMs: 0, freshMs: FRESH, retentionMs: FRESH }, phase: phaseOf[shadow.phase], timedOut: shadow.abandoned, + logging: shadow.accepted.logging, diffHook: shadow.diffHook, } : undefined; const reading = live && (shadow.phase === C0_READ || shadow.phase === CONFIRMATION_READ); const decoding = live && shadow.phase === SNAPSHOT_DECODE; return { remoteValues: [codeOf[shadow.frame]], created: [shadow.created], watermark: [shadow.watermark], now: shadow.now, skew: shadow.wall - shadow.now, - readFailed: shadow.readFailed, loadFailed: shadow.loadFailed, dumpFailed: shadow.dumpFailed, writeFailed: shadow.writeFailed, jobs: job ? [job] : [], + readFailed: shadow.readFailed, loadFailed: shadow.loadFailed, dumpFailed: shadow.dumpFailed, writeFailed: shadow.writeFailed, + loggerFailed: shadow.loggerFailed, diffHook: shadow.diffHook, defaultLogging: shadow.defaultLogging, overlayLogging: shadow.overlayLogging, jobs: job ? [job] : [], reads: reading ? [{ read: o.reads - 1, flight: source }] : [], loads: decoding ? [{ load: o.loads - 1, flight: source }] : [] }; } -const layoutFields = ["remoteValues", "created", "watermark", "now", "skew", "readFailed", "loadFailed", "dumpFailed", "writeFailed", "jobs", "reads", "loads"]; +const layoutFields = ["remoteValues", "created", "watermark", "now", "skew", "readFailed", "loadFailed", "dumpFailed", "writeFailed", + "loggerFailed", "diffHook", "defaultLogging", "overlayLogging", "jobs", "reads", "loads"]; const binding = fidelityBinding({ publicChannels: ["o", "d"], layoutFields, view: (shadow, state) => [modelView(shadow, shadow.observed), { ...state, reads: state.reads.map(held => ({ read: held.read, flight: held.flight })), loads: state.loads.map(held => ({ load: held.load, flight: held.flight })) }] }); @@ -193,8 +278,7 @@ export function shadowWitnesses(histories, recorder = createWitnessRecorder()) { if (binding.carriesPrivateState(predictions)) binding.checkFidelity(shadows, predictions, path); // -- Comparison, captured policy and verdict flow ---------------------------------- let c0Released = false, shadowTimedOut = false; - let logging = steps[0].choice >= 4 && steps[0].choice < 8; - const defaultLogging = logging; + const omittedDefaultLogging = Object.values(shadows[0].defaultLogging).every(input => input === ""); let sawShadowPolicy = false, acceptedDefaultLogging = false; let shadowPolicy = 0, jobAdmitted = false, changedAdmittedJob = false; let acceptedLogging = false, acceptedInvalidLogging = false, invalidLoggingReportedAtAdmission = false, wallRolledAfterC0 = false; @@ -206,10 +290,10 @@ export function shadowWitnesses(histories, recorder = createWitnessRecorder()) { const before = shadows[i - 1]; if (step.action === "beginCall") { c0Released = false; shadowTimedOut = false; - acceptedLogging = logging; wallRolledAfterC0 = false; - acceptedInvalidLogging = before.invalidLog === true; + acceptedLogging = logsMismatch(shadows[i].accepted.logging); wallRolledAfterC0 = false; + acceptedInvalidLogging = shadows[i].accepted.invalid; invalidLoggingReportedAtAdmission = step.diagnostics.configErrors === steps[i - 1].diagnostics.configErrors + 1; - acceptedDefaultLogging = !defaultLogging && !sawShadowPolicy; + acceptedDefaultLogging = omittedDefaultLogging && !sawShadowPolicy; jobAdmitted = o.reads > previous.reads; changedAdmittedJob = false; if (steps[0].choice === 11 && jobAdmitted && o.calls.at(-1) === 0 && o.shadow.length === previous.shadow.length) { recorder.credit("source-work-before-deadline-dispatches-read"); @@ -225,11 +309,12 @@ export function shadowWitnesses(histories, recorder = createWitnessRecorder()) { } } if (step.action === "logPolicy" || step.action === "shadowPolicy") sawShadowPolicy = true; - if (step.action === "logPolicy") { logging = step.choice === 1; shadowPolicy = 0; } + if (step.action === "logPolicy") shadowPolicy = 0; if (step.action === "shadowPolicy") { - shadowPolicy = step.choice; logging = defaultLogging; + shadowPolicy = step.choice; if (jobAdmitted && step.choice > 0) changedAdmittedJob = true; } + const logging = logsMismatch(capturedLogging(shadows[i].defaultLogging, shadows[i].overlayLogging).logging); if (step.action === "rollbackWall" && c0Released) wallRolledAfterC0 = true; if (steps[0].choice === 12 && ["resolveLoader", "rejectLoader"].includes(step.action) && previous.calls.at(-1) === 0 && o.calls.at(-1) === 4 && o.reads === previous.reads && @@ -259,6 +344,7 @@ export function shadowWitnesses(histories, recorder = createWitnessRecorder()) { if (outcome === "match") recorder.credit("enabled-logging-match-has-no-warning"); if (outcome === "superseded") recorder.credit("enabled-logging-superseded-has-no-warning"); } + if (outcome === "mismatch") warningLabels(recorder, step, steps[i - 1], before, shadows[i], steps[0].choice, invalidLoggingReportedAtAdmission); if (step.diagnostics.ages.length > steps[i - 1].diagnostics.ages.length) { if (wallRolledAfterC0 && step.diagnostics.ages.at(-1) === 0) recorder.credit("age-clamped-after-rollback"); if (step.diagnostics.ages.at(-1) > 0) recorder.credit("age-at-verdict"); diff --git a/formal/tools/check-semantic-coverage.mjs b/formal/tools/check-semantic-coverage.mjs index 86081b06..07651d74 100644 --- a/formal/tools/check-semantic-coverage.mjs +++ b/formal/tools/check-semantic-coverage.mjs @@ -23,7 +23,7 @@ export function checkProfiles(registry = parse('formal/catalogs/profiles.json')) read(registry.normativeDefinition); if (registry.behavioralAuthority?.kind !== 'quint' || registry.behavioralAuthority.executionManifest !== 'formal/catalogs/execution.json' || !registry.behavioralAuthority.conflictPolicy) throw new Error('Quint behavioral authority must be explicit'); for (const profile of registry.profiles) { - const supportedVersion = ["effects", "policy", "shadow"].includes(profile.id) ? 3 : ["dark-layers", "scope", "layers", "independent"].includes(profile.id) ? 2 : 1; + const supportedVersion = profile.id === "shadow" ? 4 : ["effects", "policy"].includes(profile.id) ? 3 : ["dark-layers", "scope", "layers", "independent"].includes(profile.id) ? 2 : 1; if (profile.version !== supportedVersion) throw new Error(`${profile.id}: unsupported profile version`); read(profile.definition); read(profile.model); for (const path of profile.witnessSources ?? []) read(path); diff --git a/formal/tools/conformance-bindings.mjs b/formal/tools/conformance-bindings.mjs index d36884a4..62481433 100644 --- a/formal/tools/conformance-bindings.mjs +++ b/formal/tools/conformance-bindings.mjs @@ -8,6 +8,7 @@ const protocolTitles = { invalidTimestampVectors: 'rejects timestamp: ', envelopeVectors: 'handles payload envelope: ', compressedDecodeVectors: 'decodes compressed payload: ', compressionWriteVectors: 'selects compression representation: ', durationVectors: 'bounds physical duration: ', rampVectors: 'assigns deterministic cohort: ', + mismatchPathVectors: 'computes mismatch paths: ', }; export const protocolGoRoots = { keyVectors: 'TestProtocolKeys', invalidKeyVectors: 'TestProtocolKeys', frameVectors: 'TestProtocolFrames', @@ -16,6 +17,7 @@ export const protocolGoRoots = { normalizeArgsVectors: 'TestProtocolRemainingVectors', invalidTimestampVectors: 'TestProtocolRemainingVectors', envelopeVectors: 'TestProtocolRemainingVectors', compressedDecodeVectors: 'TestProtocolRemainingVectors', compressionWriteVectors: 'TestProtocolRemainingVectors', durationVectors: 'TestProtocolRemainingVectors', + mismatchPathVectors: 'TestProtocolRemainingVectors', }; const goName = name => { if (!name || /[^\x20-\x7e]/.test(name) || name.includes('#')) throw new Error('Unsupported Go fixture test name'); diff --git a/formal/tools/validation.mjs b/formal/tools/validation.mjs index 16824da4..0a146b4c 100644 --- a/formal/tools/validation.mjs +++ b/formal/tools/validation.mjs @@ -18,7 +18,7 @@ export const targetDescriptions = { check: 'TypeScript, Go, Rust and Python checks, docs build and reviewed inventories; no Quint generation or Docker', 'check-ts': 'Typecheck, unit coverage, build and packed-package checks on Node 24', 'check-go': 'Go vet, formatting check and default tests with race detection and coverage (go/coverage-unit.out)', - 'check-rust': 'Rust formatting check, clippy with warnings denied and default tests including the smoke conformance run', + 'check-rust': 'Rust formatting check, clippy with warnings denied (all targets and features, then the published library without test-util) and default tests including the smoke conformance run', 'check-python': 'Python native, protocol, scenario and committed smoke tests with coverage (coverage/python/native.lcov)', docs: 'Check shared examples and links; generate native API references and the documentation site (Go, Rust and Python required)', audit: 'Check source, behavior, feature, Go, generated-fixture and generated-view freshness inventories, the Quint style caps and the course citations', @@ -182,8 +182,13 @@ export function validationPlan(target, { directory = root, environment = process 'check-ts': [pnpm('Typecheck TypeScript', 'typecheck'), pnpm('Run TypeScript unit tests with coverage', 'test'), pnpm('Build package', 'build'), pnpm('Check packed package on Node 24', 'test:package')], 'check-go': [go('Go vet', 'vet', './...'), { label: 'Check Go formatting', command: 'gofmt', args: ['-l', 'go'], requireEmptyStdout: true }, nativeGo(false, { coverage: true })], + // Every other Rust step turns on test-util (--all-features, and the crate's + // dev-dependency on itself), which replaces shipped code such as the + // shadow-warning JSON renderer. The library is therefore linted once more + // with every published feature and without test-util, as applications build it. 'check-rust': [cargo('Check Rust formatting', 'fmt', '--check'), cargo('Lint Rust with clippy', 'clippy', '--all-targets', '--all-features', '--', '-D', 'warnings'), + cargo('Lint the published Rust library without test-util', 'clippy', '--lib', '--features', 'redis,prometheus', '--', '-D', 'warnings'), cargo('Run Rust default tests', 'test', '--all-features')], 'check-python': [{ label: 'Clear prior Python native coverage', remove: ['coverage/python/.coverage-native', 'coverage/python/native.lcov'] }, python('Run Python native, wire, scenario and smoke tests with coverage', '-m', 'coverage', 'run', diff --git a/go/README.md b/go/README.md index e2807029..1f7068e2 100644 --- a/go/README.md +++ b/go/README.md @@ -180,7 +180,8 @@ receives the same verdict event the observers do, so wire an exporter through one of those paths to avoid double-counting it. `WithRecoveryOutcomes` is an optional separate recovery hook. Prometheus and DogStatsD adapters preserve metric names, labels, units and buckets; logical -keys never become labels. `WithLogger` replaces the standard logger; either is +keys never become labels. `WithLogger` replaces the standard logger, which +writes each entry on one line with control characters escaped; either is failure-isolated. Mismatch logging is opt-in, confirmed, and bounded. Prometheus adapters reuse collectors created by an earlier adapter with the diff --git a/go/api.go b/go/api.go index 8177e069..cf58d6f6 100644 --- a/go/api.go +++ b/go/api.go @@ -118,6 +118,33 @@ type Operation[T any] struct { ShouldRecover RecoveryPredicate // Comparator decides shadow matches; nil compares with SemanticEqual. Comparator func(cached, source T) (bool, error) + // MismatchDiff replaces the built-in JSON difference computation behind + // the mismatchPaths and diffJson fields of confirmed-mismatch warnings; it + // runs only when the resolved mismatch-logging diff level is above + // MismatchDiffOff. Inputs are borrowed snapshots. Like Comparator, it must + // be deterministic, side-effect-free, non-mutating and bounded. A returned + // error or panic logs null for the paths and the diff, without falling back + // to the built-in computation; a nil slice is an empty list. + MismatchDiff func(cached, source T) ([]MismatchEntry, error) +} + +// MismatchEntry is one differing field of a confirmed shadow mismatch: an RFC +// 6901 JSON Pointer into the compared value and the value on each side there. +// Absent omits a side the field does not have, as the built-in computation +// does for a member present on one side only; set both sides to Absent when +// only the location is known. A nil side is JSON null. +// +// A side renders as JSONCodec writes it, except that a value implementing +// json.Marshaler, json.RawMessage included, renders at any depth as its +// MarshalJSON output parsed back, so a type that redacts itself stays redacted +// in diffJson. As in encoding/json, a pointer's MarshalJSON runs only for an +// addressable value, such as a slice element. A side with no rendering (a +// cycle, a *big.Int, a function, or a MarshalJSON that fails, panics or +// returns output that is not JSON or holds a number outside double range) +// logs a null diffJson with render_error and keeps the paths. +type MismatchEntry struct { + Path string + Cached, Source any } // Event records public diagnostics; Data contains the backend-neutral labels. diff --git a/go/behavior_driver_test.go b/go/behavior_driver_test.go index 4adcca6e..6126fa6a 100644 --- a/go/behavior_driver_test.go +++ b/go/behavior_driver_test.go @@ -717,6 +717,11 @@ func (d *behaviorDriver) apply(input obj) error { return comparator == "equal", nil } } + if _, hooked := d.fixture["mismatchDiff"]; hooked { + operation.MismatchDiff = func(any, any) ([]MismatchEntry, error) { + return nil, errors.New("controlled mismatch diff failure") + } + } call := func(callctx context.Context) error { budget := int64(60000) if operation.SourceTimeout > 0 { @@ -749,6 +754,8 @@ func (d *behaviorDriver) apply(input obj) error { result = obj{"status": "error", "error": d.classifyError(err)} } else if IsAbsent(value) { result["value"] = obj{"absent": true} + } else if value == any(behaviorUnrenderable) { + result["value"] = obj{"unrenderable": true} } d.mu.Lock() ba(d.observed["calls"])[index] = result @@ -784,6 +791,8 @@ func (d *behaviorDriver) apply(input obj) error { d.sourceErrors[index] = &FallbackTimeoutError{} } gate.err = d.sourceErrors[index] + } else if bb(input["unrenderable"]) { + gate.value = behaviorUnrenderable } else { gate.value = bdefault(input, "value", Absent) } @@ -1044,13 +1053,60 @@ func (l behaviorLogger) fail() { } func (l behaviorLogger) Debug(string, any) { l.fail() } func (l behaviorLogger) Error(string, any) { l.fail() } + +// Warn records a mismatch warning before it fails: the faults command's logger +// fault makes every Warn panic, the one way a Go logger fails, while Debug, +// Error and the metric observers keep working. func (l behaviorLogger) Warn(message string, details any) { if message == "DialCache shadow validation mismatch" { - l.d.record("mismatchWarning", bm(details)) + l.d.record("mismatchWarning", behaviorWarningObservation(bm(details))) } l.fail() + if l.d.fault("logger") { + panic("controlled logger failure") + } } +// behaviorWarningObservation is the warning's fields as logged, each present +// exactly when the warning carries it. The mismatchPaths slice crosses the wire +// as one compact JSON string, so every observed field stays a primitive +// (PORTING.md, mismatchWarning); only an untyped nil, the library's failed +// field, is recorded as null. +func behaviorWarningObservation(details obj) obj { + fields := obj{} + for name, value := range details { + if name != "mismatchPaths" { + fields[name] = value + continue + } + if value == nil { + fields["mismatchPathsJson"] = nil + continue + } + var encoded strings.Builder + encoder := json.NewEncoder(&encoded) + encoder.SetEscapeHTML(false) + // The library logs the paths as a []string, which always encodes. + _ = encoder.Encode(value) + fields["mismatchPathsJson"] = strings.TrimSuffix(encoded.String(), "\n") + } + return fields +} + +// behaviorUnrenderable is the source result with no JSON rendering (D13): its +// MarshalJSON fails, so every JSON rendering of it fails, the driver codec's +// included. The driver knows it by identity and observes a call that returns +// it as {"unrenderable": true}. The method has a value receiver because the +// library renders a struct through encoding/json after dereferencing it, and +// the struct is not zero-size, so the pointer is distinct from every other. +type behaviorUnrenderableValue struct{ name string } + +func (behaviorUnrenderableValue) MarshalJSON() ([]byte, error) { + return nil, errors.New("controlled unrenderable value") +} + +var behaviorUnrenderable = &behaviorUnrenderableValue{name: "unrenderable"} + func (r behaviorRemote) Read(ctx context.Context, key, watermark string) (ReadResult, error) { d := r.d index := d.increment("reads") diff --git a/go/behavior_registry_test.go b/go/behavior_registry_test.go index f7ea2542..a711384b 100644 --- a/go/behavior_registry_test.go +++ b/go/behavior_registry_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -var behaviorProfileVersions = map[string]int64{"shadow-read-deadlines": 1, "dark-layers": 2, "recovery-read": 1, "local-failure": 1, "runtime-boundaries": 1, "shadow-layers": 1, "local-clock": 1, "source-budgets": 1, "effects": 3, "scope": 2, "policy": 3, "layers": 2, "recovery": 1, "independent": 2, "shadow": 3, "admission": 1} +var behaviorProfileVersions = map[string]int64{"shadow-read-deadlines": 1, "dark-layers": 2, "recovery-read": 1, "local-failure": 1, "runtime-boundaries": 1, "shadow-layers": 1, "local-clock": 1, "source-budgets": 1, "effects": 3, "scope": 2, "policy": 3, "layers": 2, "recovery": 1, "independent": 2, "shadow": 4, "admission": 1} func validateBehaviorProfileRegistry(raw []byte, name string, version int64) error { decoded, err := behaviorJSON(raw) diff --git a/go/codec.go b/go/codec.go index f0a1d636..c9412b11 100644 --- a/go/codec.go +++ b/go/codec.go @@ -47,7 +47,7 @@ func (JSONCodec[T]) Encode(value T) (Payload, error) { if IsAbsent(value) { return Payload{Bytes: []byte(JSONUndefinedSentinel)}, nil } - raw, err := appendJSON(nil, reflect.ValueOf(value), make(map[visit]bool)) + raw, err := appendJSON(nil, reflect.ValueOf(value), make(map[visit]bool), false) return Payload{Bytes: raw}, err } func (JSONCodec[T]) Decode(payload Payload) (T, error) { @@ -219,7 +219,13 @@ func quoteJSON(dst []byte, value string) []byte { } return append(dst, '"') } -func appendJSON(dst []byte, value reflect.Value, seen map[visit]bool) ([]byte, error) { + +// appendJSON writes value in the portable JSON encoding. With marshalers set, a +// value at any depth that implements json.Marshaler renders as its MarshalJSON +// output parsed back, while Absent and big integers keep their own rules. +// JSONCodec leaves it unset, so its stored encoding reaches MarshalJSON only +// through a struct. +func appendJSON(dst []byte, value reflect.Value, seen map[visit]bool, marshalers bool) ([]byte, error) { if !value.IsValid() { return append(dst, "null"...), nil } @@ -247,6 +253,11 @@ func appendJSON(dst []byte, value reflect.Value, seen map[visit]bool) ([]byte, e if _, ok := value.Interface().(big.Int); ok { return nil, errors.New("JSON cannot encode a bigint") } + if marshalers { + if marshaler, ok := jsonMarshaler(value); ok { + return appendReparsed(dst, marshaler, seen) + } + } if object, ok := value.Interface().(JSONObject); ok { key := visit{value.Kind(), value.Pointer()} if seen[key] { @@ -254,14 +265,14 @@ func appendJSON(dst []byte, value reflect.Value, seen map[visit]bool) ([]byte, e } seen[key] = true defer delete(seen, key) - return appendObject(dst, object, seen) + return appendObject(dst, object, seen, marshalers) } if binary, ok := value.Interface().([]byte); ok { data := make([]any, len(binary)) for i, b := range binary { data[i] = b } - return appendObject(dst, JSONObject{{"type", "Buffer"}, {"data", data}}, seen) + return appendObject(dst, JSONObject{{"type", "Buffer"}, {"data", data}}, seen, marshalers) } } if number, ok := scalarNumber(value); ok { @@ -293,7 +304,7 @@ func appendJSON(dst []byte, value reflect.Value, seen map[visit]bool) ([]byte, e dst = append(dst, ',') } var err error - dst, err = appendJSON(dst, value.Index(i), seen) + dst, err = appendJSON(dst, value.Index(i), seen, marshalers) if err != nil { return nil, err } @@ -318,28 +329,54 @@ func appendJSON(dst []byte, value reflect.Value, seen map[visit]bool) ([]byte, e for _, name := range names { object = append(object, JSONMember{name.String(), value.MapIndex(name).Interface()}) } - return appendObject(dst, object, seen) + return appendObject(dst, object, seen, marshalers) case reflect.Struct: // Native structs opt into Go's public JSON field/tag conventions, then the // resulting JSON domain uses the same portable number/string encoding. - raw, err := json.Marshal(value.Interface()) - if err != nil { - return nil, err + // A hook side marshals an addressable struct through its pointer, so a + // field's pointer-receiver MarshalJSON runs as encoding/json runs it. + if marshalers && value.CanAddr() { + return appendReparsed(dst, value.Addr().Interface(), seen) } - var object any - if err = json.Unmarshal(raw, &object); err != nil { - return nil, err - } - return appendJSON(dst, reflect.ValueOf(object), seen) + return appendReparsed(dst, value.Interface(), seen) default: return nil, fmt.Errorf("unsupported JSON value %s", value.Kind()) } } + +// jsonMarshaler returns the json.Marshaler that encoding/json calls for value: +// its own, or its pointer's when value is addressable. +func jsonMarshaler(value reflect.Value) (json.Marshaler, bool) { + if marshaler, ok := value.Interface().(json.Marshaler); ok { + return marshaler, true + } + if !value.CanAddr() { + return nil, false + } + marshaler, ok := value.Addr().Interface().(json.Marshaler) + return marshaler, ok +} + +// appendReparsed renders value with encoding/json and parses the output back, +// never splicing it, so only its JSON domain reaches dst in the portable +// encoding. MarshalJSON output that is not JSON, or that holds a number outside +// double range, has no rendering. +func appendReparsed(dst []byte, value any, seen map[visit]bool) ([]byte, error) { + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + var parsed any + if err = json.Unmarshal(raw, &parsed); err != nil { + return nil, err + } + return appendJSON(dst, reflect.ValueOf(parsed), seen, false) +} func arrayIndex(name string) (uint64, bool) { n, e := strconv.ParseUint(name, 10, 32) return n, e == nil && n < 4294967295 && strconv.FormatUint(n, 10) == name } -func appendObject(dst []byte, object JSONObject, seen map[visit]bool) ([]byte, error) { +func appendObject(dst []byte, object JSONObject, seen map[visit]bool, marshalers bool) ([]byte, error) { // Repeated properties replace their value without moving their first position. unique := make(JSONObject, 0, len(object)) indices := map[string]int{} @@ -372,7 +409,7 @@ func appendObject(dst []byte, object JSONObject, seen map[visit]bool) ([]byte, e dst = quoteJSON(dst, member.Name) dst = append(dst, ':') var err error - dst, err = appendJSON(dst, reflect.ValueOf(member.Value), seen) + dst, err = appendJSON(dst, reflect.ValueOf(member.Value), seen, marshalers) if err != nil { return nil, err } diff --git a/go/internal/logging/logging.go b/go/internal/logging/logging.go index 82ab3ba5..c99de307 100644 --- a/go/internal/logging/logging.go +++ b/go/internal/logging/logging.go @@ -1,12 +1,79 @@ // Package logging supplies the default cache logger. package logging -import "log" +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "strconv" + "strings" + "unicode/utf8" +) // The default follows the TypeScript console logger. Applications may provide // their own Logger; all its callbacks are isolated from cache/source results. type Logger struct{} -func (Logger) Debug(message string, details any) { log.Print(message, ": ", details) } -func (Logger) Warn(message string, details any) { log.Print(message, ": ", details) } -func (Logger) Error(message string, details any) { log.Print(message, ": ", details) } +func (Logger) Debug(message string, details any) { write(message, details) } +func (Logger) Warn(message string, details any) { write(message, details) } +func (Logger) Error(message string, details any) { write(message, details) } + +// write logs one line. Details can carry application text, such as the member +// names in a mismatch warning's pointers, so the whole line is escaped: a +// newline cannot forge another line and an escape sequence cannot reach the +// terminal. +func write(message string, details any) { + log.Print(escapeControls(message + ": " + renderDetails(details))) +} + +// renderDetails writes a structured details map, such as the mismatch +// warning's, as JSON, so each string keeps its bounds: a root pointer "" prints +// as [""] where an empty list prints as [], and a pointer holding a space stays +// one element. Every escape is a JSON escape, so the text still parses after +// escapeControls: encoding/json escapes C0 controls, U+2028 and U+2029 and +// replaces bytes that are not UTF-8 with U+FFFD, DEL is written as \u007f here, +// and escapeControls writes C1 controls as \u0080 to \u009f. An error, or any +// other details value, prints as fmt.Sprint does. +func renderDetails(details any) string { + fields, ok := details.(map[string]any) + if !ok { + return fmt.Sprint(details) + } + var encoded bytes.Buffer + encoder := json.NewEncoder(&encoded) + encoder.SetEscapeHTML(false) + // Only the library logs here, and its warning fields (strings, finite + // numbers, string lists and nulls) always encode. + _ = encoder.Encode(fields) + return strings.ReplaceAll(strings.TrimSuffix(encoded.String(), "\n"), "\x7f", `\u007f`) +} + +// escapeControls writes each C0 and C1 control character, DEL, U+2028, U+2029 +// and each byte that is not valid UTF-8 as its Go escape sequence, such as \n, +// \x1b, \u0085 or \u2028, and leaves every other character as written. +func escapeControls(text string) string { + var escaped strings.Builder + start := 0 + for index := 0; index < len(text); { + r, size := utf8.DecodeRuneInString(text[index:]) + if (r != utf8.RuneError || size != 1) && !isControl(r) { + index += size + continue + } + quoted := strconv.Quote(text[index : index+size]) + escaped.WriteString(text[start:index]) + escaped.WriteString(quoted[1 : len(quoted)-1]) + index += size + start = index + } + if start == 0 { + return text + } + escaped.WriteString(text[start:]) + return escaped.String() +} + +func isControl(r rune) bool { + return r < 0x20 || r >= 0x7f && r <= 0x9f || r == '\u2028' || r == '\u2029' +} diff --git a/go/internal/logging/logging_test.go b/go/internal/logging/logging_test.go new file mode 100644 index 00000000..8c9f9b96 --- /dev/null +++ b/go/internal/logging/logging_test.go @@ -0,0 +1,101 @@ +package logging + +import ( + "bytes" + "encoding/json" + "errors" + "log" + "reflect" + "strings" + "testing" +) + +// capture returns what the default logger writes while run executes. +func capture(t *testing.T, run func()) string { + t.Helper() + var output bytes.Buffer + writer, flags := log.Writer(), log.Flags() + log.SetOutput(&output) + log.SetFlags(0) + t.Cleanup(func() { log.SetOutput(writer); log.SetFlags(flags) }) + run() + return output.String() +} + +func TestDefaultLoggerKeepsAWarningOnOneLine(t *testing.T) { + // A member name reaches the warning's pointers as written. + details := map[string]any{"mismatchPaths": []string{"/user\nDialCache forged: line\x1b[2J"}, "mismatchPathCount": 1} + for name, write := range map[string]func(string, any){"Debug": Logger{}.Debug, "Warn": Logger{}.Warn, "Error": Logger{}.Error} { + t.Run(name, func(t *testing.T) { + output := capture(t, func() { write("DialCache shadow validation mismatch", details) }) + want := `DialCache shadow validation mismatch: {"mismatchPathCount":1,"mismatchPaths":["/user\nDialCache forged: line\u001b[2J"]}` + "\n" + if output != want { + t.Fatalf("logged %q, want %q", output, want) + } + }) + } +} + +func TestDefaultLoggerWritesDetailsAsJSON(t *testing.T) { + for _, test := range []struct { + name string + details any + want string + }{ + {"a root pointer", map[string]any{"mismatchPaths": []string{""}}, `{"mismatchPaths":[""]}`}, + {"no pointers", map[string]any{"mismatchPaths": []string{}}, `{"mismatchPaths":[]}`}, + {"a pointer holding a space", map[string]any{"mismatchPaths": []string{"/a b"}}, `{"mismatchPaths":["/a b"]}`}, + {"two pointers", map[string]any{"mismatchPaths": []string{"/a", "b"}}, `{"mismatchPaths":["/a","b"]}`}, + {"null and text fields", map[string]any{"diffJson": nil, "cachedValueJson": `{"a":"<&>"}`, "cachedValueAgeSeconds": 30.5}, + `{"cachedValueAgeSeconds":30.5,"cachedValueJson":"{\"a\":\"<&>\"}","diffJson":null}`}, + {"an error", errors.New("redis: connection refused"), "redis: connection refused"}, + } { + t.Run(test.name, func(t *testing.T) { + output := capture(t, func() { Logger{}.Warn("message", test.details) }) + if want := "message: " + test.want + "\n"; output != want { + t.Fatalf("logged %q, want %q", output, want) + } + }) + } +} + +func TestDefaultLoggerDetailsParseBack(t *testing.T) { + // encoding/json leaves DEL and C1 controls raw; both must still leave the + // line one line whose details parse as JSON. + name := "/a\x7fb\u0085c\u2028d\ne\x1bf\u009fg" + details := map[string]any{"mismatchPaths": []string{name, "/h\xffi"}} + output := capture(t, func() { Logger{}.Warn("message", details) }) + text, found := strings.CutPrefix(strings.TrimSuffix(output, "\n"), "message: ") + if !found || strings.ContainsAny(text, "\n\r\x1b\x7f\u0085\u009f\u2028") { + t.Fatalf("logged %q", output) + } + var parsed map[string][]string + if err := json.Unmarshal([]byte(text), &parsed); err != nil { + t.Fatalf("details %q do not parse: %v", text, err) + } + if want := []string{name, "/h\ufffdi"}; !reflect.DeepEqual(parsed["mismatchPaths"], want) { + t.Fatalf("parsed %q, want %q", parsed["mismatchPaths"], want) + } +} + +func TestEscapeControls(t *testing.T) { + for _, test := range []struct{ name, text, want string }{ + {"readable text unchanged", `id "é" ~/a\b 💡`, `id "é" ~/a\b 💡`}, + {"C0 controls", "a\x00b\tc\rd\ne\x1bf\x1f", `a\x00b\tc\rd\ne\x1bf\x1f`}, + {"DEL", "a\x7fb", `a\x7fb`}, + {"C1 controls", "a\u0080b\u0085c\u009bd\u009f", `a\u0080b\u0085c\u009bd\u009f`}, + {"line and paragraph separators", "a\u2028b\u2029c", `a\u2028b\u2029c`}, + {"bytes that are not UTF-8", "a\x9bb\xffc\xe2\x80", `a\x9bb\xffc\xe2\x80`}, + {"replacement character kept", "a\ufffdb", "a\ufffdb"}, + {"leading and trailing controls", "\nab\n", `\nab\n`}, + } { + t.Run(test.name, func(t *testing.T) { + if got := escapeControls(test.text); got != test.want { + t.Fatalf("escaped %q as %q, want %q", test.text, got, test.want) + } + if got := escapeControls(test.text); strings.ContainsAny(got, "\n\r\x1b\u2028\u2029\u0085") { + t.Fatalf("escaped text %q keeps a control character", got) + } + }) + } +} diff --git a/go/internal/testsuite/policy_test.go b/go/internal/testsuite/policy_test.go index aac49152..ead84b1c 100644 --- a/go/internal/testsuite/policy_test.go +++ b/go/internal/testsuite/policy_test.go @@ -30,7 +30,7 @@ func TestPolicySparseResolutionAndSnapshots(t *testing.T) { if err != nil { t.Fatal(err) } - if !resolved.RequestLocal || resolved.Coalesce || resolved.Local.Enabled || resolved.Local.Reason != "ramped_down" || !resolved.Local.Configured || resolved.Local.TTL != time.Second || !resolved.Remote.Enabled || resolved.Remote.TTL != 4*time.Second || resolved.StaleOnErrorMaxAge != 5*time.Second || resolved.RemoteReadTimeout != 30*time.Millisecond || !resolved.Shadow.Enabled || resolved.Shadow.LogMismatches { + if !resolved.RequestLocal || resolved.Coalesce || resolved.Local.Enabled || resolved.Local.Reason != "ramped_down" || !resolved.Local.Configured || resolved.Local.TTL != time.Second || !resolved.Remote.Enabled || resolved.Remote.TTL != 4*time.Second || resolved.StaleOnErrorMaxAge != 5*time.Second || resolved.RemoteReadTimeout != 30*time.Millisecond || !resolved.Shadow.Enabled || resolved.Shadow.LogValue || resolved.Shadow.LogDiff != MismatchDiffOff || resolved.Shadow.LoggingConfigError { t.Fatalf("wrong sparse policy: %+v", resolved) } copy := SnapshotPolicy(base) @@ -57,6 +57,12 @@ func TestPolicyStaticValidation(t *testing.T) { map[string]any{"staleOnErrorMaxAgeSec": 2}, map[string]any{"ttlSec": map[string]any{"remote": 2}, "staleOnErrorMaxAgeSec": 2}, map[string]any{"shadow": map[string]any{"logMismatches": 1}}, + map[string]any{"shadow": map[string]any{"mismatchLogging": nil}}, + map[string]any{"shadow": map[string]any{"mismatchLogging": "paths"}}, + map[string]any{"shadow": map[string]any{"mismatchLogging": map[string]any{"value": "true"}}}, + map[string]any{"shadow": map[string]any{"mismatchLogging": map[string]any{"value": nil}}}, + map[string]any{"shadow": map[string]any{"mismatchLogging": map[string]any{"diff": "full"}}}, + map[string]any{"shadow": map[string]any{"mismatchLogging": map[string]any{"diff": true}}}, } { if _, err := ParsePolicy(config); err == nil { t.Fatalf("accepted invalid static config: %#v", config) @@ -68,11 +74,23 @@ func TestPolicyStaticValidation(t *testing.T) { if err := ValidatePolicy(Policy{LocalTTL: time.Duration(1500) * time.Millisecond}); err == nil { t.Fatal("accepted fractional second TTL") } + if err := ValidatePolicy(Policy{Shadow: &ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: Ptr(MismatchDiffLevel("full"))}}}); err == nil { + t.Fatal("accepted an unknown static diff level") + } + parsed, err := ParsePolicy(map[string]any{"shadow": map[string]any{"mismatchLogging": map[string]any{"value": false, "diff": MismatchDiffPaths}}}) + if err != nil || parsed.Shadow.MismatchLogging == nil || *parsed.Shadow.MismatchLogging.Value || *parsed.Shadow.MismatchLogging.Diff != MismatchDiffPaths { + t.Fatalf("static mismatch-logging leaves: %+v %v", parsed.Shadow, err) + } + snapshot := SnapshotPolicy(parsed) + *parsed.Shadow.MismatchLogging.Value, *parsed.Shadow.MismatchLogging.Diff = true, MismatchDiffValues + if *snapshot.Shadow.MismatchLogging.Value || *snapshot.Shadow.MismatchLogging.Diff != MismatchDiffPaths { + t.Fatal("static snapshot retained mutable mismatch-logging leaves") + } } func TestRuntimePolicyFailureScopes(t *testing.T) { base := Policy{RequestLocal: true, LocalTTL: time.Duration(1000) * time.Millisecond, RemoteTTL: time.Duration(2000) * time.Millisecond} - for _, overlay := range []any{false, []any{}, map[string]any{"ttlSec": nil}, map[string]any{"shadow": nil}, map[string]any{"requestLocal": nil}, map[string]any{"coalesce": 1}, map[string]any{"remoteReadTimeoutMs": 0}} { + for _, overlay := range []any{false, []any{}, map[string]any{"ttlSec": nil}, map[string]any{"shadow": nil}, map[string]any{"shadow": map[string]any{"mismatchLogging": nil}}, map[string]any{"shadow": map[string]any{"mismatchLogging": []any{}}}, map[string]any{"shadow": map[string]any{"mismatchLogging": "paths"}}, map[string]any{"shadow": map[string]any{"logMismatches": true, "mismatchLogging": 1}}, map[string]any{"requestLocal": nil}, map[string]any{"coalesce": 1}, map[string]any{"remoteReadTimeoutMs": 0}} { if _, err := ResolvePolicy(base, RawPolicy(overlay), policyTestIdentity(), PolicyDefaults{}); err == nil { t.Fatalf("accepted invalid invocation policy: %#v", overlay) } @@ -103,7 +121,7 @@ func TestRuntimePolicyFailureScopes(t *testing.T) { func TestRecoveryRetentionAndShadowDiagnosticsRemainIndependent(t *testing.T) { base := Policy{RemoteTTL: time.Duration(1000) * time.Millisecond, RemoteRamp: policyTestPtr(0.0), StaleOnErrorMaxAge: Ptr(24 * time.Hour)} r, err := ResolvePolicy(base, JSONPolicy{"shadow": map[string]any{"ramp": 100, "logMismatches": "invalid"}}, policyTestIdentity(), PolicyDefaults{}) - if err != nil || r.Remote.Enabled || !r.Remote.Configured || r.StaleOnErrorMaxAge != 24*time.Hour || !r.Shadow.Enabled || r.Shadow.ConfigError || !r.Shadow.LoggingConfigError || r.Shadow.LogMismatches { + if err != nil || r.Remote.Enabled || !r.Remote.Configured || r.StaleOnErrorMaxAge != 24*time.Hour || !r.Shadow.Enabled || r.Shadow.ConfigError || !r.Shadow.LoggingConfigError || r.Shadow.LogValue || r.Shadow.LogDiff != MismatchDiffOff { t.Fatalf("wrong independent options: %+v %v", r, err) } r, err = ResolvePolicy(base, JSONPolicy{"shadow": map[string]any{"ramp": nil}}, policyTestIdentity(), PolicyDefaults{}) @@ -126,3 +144,194 @@ func TestJSONPolicyIsAnObjectWhereverItAppears(t *testing.T) { t.Fatalf("ParsePolicy rejected JSONPolicy containers: %v", err) } } + +func TestMismatchLoggingRecordedRows(t *testing.T) { + resolve := func(t *testing.T, static, runtime map[string]any) ResolvedShadow { + t.Helper() + var shadow any = Absent + if static != nil { + shadow = static + } + base, err := ParsePolicy(map[string]any{"shadow": shadow}) + if err != nil { + t.Fatal(err) + } + var overlay RuntimePolicy + if runtime != nil { + overlay = JSONPolicy{"shadow": runtime} + } + resolved, err := ResolvePolicy(base, overlay, policyTestIdentity(), PolicyDefaults{}) + if err != nil { + t.Fatal(err) + } + return resolved.Shadow + } + paths, valueFalse := map[string]any{"diff": "paths"}, map[string]any{"value": false} + for _, row := range []struct { + name string + static, runtime map[string]any + value bool + diff MismatchDiffLevel + invalid bool + }{ + {"a static true shorthand beside value false", map[string]any{"logMismatches": true, "mismatchLogging": valueFalse}, nil, false, MismatchDiffValues, false}, + {"a runtime true shorthand beside value false", nil, map[string]any{"logMismatches": true, "mismatchLogging": valueFalse}, false, MismatchDiffValues, false}, + {"a static false shorthand beside diff paths", map[string]any{"logMismatches": false, "mismatchLogging": paths}, nil, false, MismatchDiffPaths, false}, + {"a runtime false shorthand beside diff paths", nil, map[string]any{"logMismatches": false, "mismatchLogging": paths}, false, MismatchDiffPaths, false}, + {"a runtime false shorthand over a static diff paths", map[string]any{"mismatchLogging": paths}, map[string]any{"logMismatches": false}, false, MismatchDiffOff, false}, + {"a runtime true shorthand over a static value false", map[string]any{"mismatchLogging": valueFalse}, map[string]any{"logMismatches": true}, true, MismatchDiffValues, false}, + {"an omitted runtime leaf inheriting the static one", map[string]any{"mismatchLogging": map[string]any{"value": true, "diff": "paths"}}, map[string]any{"mismatchLogging": map[string]any{"diff": "values"}}, true, MismatchDiffValues, false}, + {"a static shorthand as a baseline under a runtime leaf", map[string]any{"logMismatches": true}, map[string]any{"mismatchLogging": paths}, true, MismatchDiffPaths, false}, + {"a malformed runtime shorthand whose same-object leaf wins", map[string]any{"mismatchLogging": map[string]any{"value": false, "diff": "values"}}, map[string]any{"logMismatches": "yes", "mismatchLogging": map[string]any{"value": true}}, true, MismatchDiffOff, true}, + {"an invalid runtime leaf beside a valid static one", map[string]any{"mismatchLogging": map[string]any{"value": true}}, map[string]any{"mismatchLogging": map[string]any{"diff": "everything"}}, true, MismatchDiffOff, true}, + {"neither layer writing a leaf", nil, map[string]any{"ramp": 100}, false, MismatchDiffOff, false}, + } { + t.Run(row.name, func(t *testing.T) { + got := resolve(t, row.static, row.runtime) + if got.LogValue != row.value || got.LogDiff != row.diff || got.LoggingConfigError != row.invalid { + t.Fatalf("resolved %+v, want value=%v diff=%s invalid=%v", got, row.value, row.diff, row.invalid) + } + }) + } +} + +// TestMismatchLoggingResolvesEachLayerExpandedFirst checks the recorded rule, +// written out without the code under test, across every combination: a +// written shorthand stands for both leaves (true: value on and diff "values"; +// false or malformed: both off), a leaf written beside it wins within its own +// layer, the runtime layer wins per leaf, an omitted leaf is off, an invalid +// leaf acts off, and any invalid input sets the one LoggingConfigError. Static +// policy stays valid, as registration guarantees. +func TestMismatchLoggingResolvesEachLayerExpandedFirst(t *testing.T) { + omit := &struct{}{} + layer := func(shorthand, group any) map[string]any { + shadow := map[string]any{} + if shorthand != omit { + shadow["logMismatches"] = shorthand + } + if group != omit { + shadow["mismatchLogging"] = group + } + return shadow + } + written := func(shadow map[string]any) (value, diff any) { + value, diff = Absent, Absent + if shorthand, present := shadow["logMismatches"]; present && !IsAbsent(shorthand) { + value, diff = shorthand == true, "off" + if shorthand == true { + diff = "values" + } + } + if group, ok := shadow["mismatchLogging"].(map[string]any); ok { + if leaf, present := group["value"]; present && !IsAbsent(leaf) { + value = leaf + } + if leaf, present := group["diff"]; present && !IsAbsent(leaf) { + diff = leaf + } + } + return value, diff + } + expected := func(static, runtime map[string]any) ResolvedShadow { + inheritedValue, inheritedDiff := written(static) + value, diff := written(runtime) + if IsAbsent(value) { + value = inheritedValue + } + if IsAbsent(diff) { + diff = inheritedDiff + } + _, isBool := value.(bool) + validValue := IsAbsent(value) || isBool + level, _ := diff.(string) + validDiff := IsAbsent(diff) || level == "off" || level == "paths" || level == "values" + shorthand, present := runtime["logMismatches"] + _, boolShorthand := shorthand.(bool) + result := ResolvedShadow{LogValue: validValue && value == true, LogDiff: MismatchDiffOff} + if validDiff && !IsAbsent(diff) { + result.LogDiff = MismatchDiffLevel(level) + } + result.LoggingConfigError = (present && !IsAbsent(shorthand) && !boolShorthand) || !validValue || !validDiff + return result + } + // typed returns the runtime group as a PolicyOverlay when every written + // value has a typed form; a nil pointer is the typed unwritten leaf. + typed := func(shadow map[string]any) (*PolicyOverlay, bool) { + policy := &ShadowPolicy{} + if shorthand, present := shadow["logMismatches"]; present && !IsAbsent(shorthand) { + flag, ok := shorthand.(bool) + if !ok { + return nil, false + } + policy.LogMismatches = &flag + } + if group, present := shadow["mismatchLogging"]; present && !IsAbsent(group) { + leaves, _ := group.(map[string]any) + policy.MismatchLogging = &MismatchLoggingPolicy{} + if leaf, present := leaves["value"]; present && !IsAbsent(leaf) { + flag, ok := leaf.(bool) + if !ok { + return nil, false + } + policy.MismatchLogging.Value = &flag + } + if leaf, present := leaves["diff"]; present && !IsAbsent(leaf) { + level, ok := leaf.(string) + if !ok { + return nil, false + } + policy.MismatchLogging.Diff = Ptr(MismatchDiffLevel(level)) + } + } + return &PolicyOverlay{Shadow: policy}, true + } + staticShorthands := []any{omit, Absent, true, false} + staticGroups := []any{ + omit, Absent, map[string]any{}, + map[string]any{"value": true}, map[string]any{"value": false}, + map[string]any{"diff": "paths"}, map[string]any{"diff": "off"}, + map[string]any{"value": true, "diff": "values"}, map[string]any{"value": Absent, "diff": "paths"}, + } + runtimeShorthands := []any{omit, Absent, true, false, "yes", nil, 0} + runtimeGroups := []any{ + omit, Absent, + map[string]any{"value": true}, map[string]any{"value": false}, + map[string]any{"diff": "paths"}, map[string]any{"diff": "values"}, + map[string]any{"value": true, "diff": "off"}, + map[string]any{"value": "yes"}, map[string]any{"value": nil}, + map[string]any{"diff": "full"}, map[string]any{"diff": "toString"}, + map[string]any{"value": Absent}, + } + combinations, typedCombinations := 0, 0 + for _, staticShorthand := range staticShorthands { + for _, staticGroup := range staticGroups { + static := layer(staticShorthand, staticGroup) + base, err := ParsePolicy(map[string]any{"shadow": static}) + if err != nil { + t.Fatal(err) + } + for _, runtimeShorthand := range runtimeShorthands { + for _, runtimeGroup := range runtimeGroups { + runtime := layer(runtimeShorthand, runtimeGroup) + want := expected(static, runtime) + overlays := []RuntimePolicy{JSONPolicy{"shadow": runtime}, RawPolicy(map[string]any{"shadow": runtime})} + if overlay, ok := typed(runtime); ok { + overlays = append(overlays, overlay) + typedCombinations++ + } + for _, overlay := range overlays { + resolved, err := ResolvePolicy(base, overlay, policyTestIdentity(), PolicyDefaults{}) + got := resolved.Shadow + if err != nil || got.LogValue != want.LogValue || got.LogDiff != want.LogDiff || got.LoggingConfigError != want.LoggingConfigError { + t.Fatalf("static %#v, runtime %#v (%T): got %+v %v, want %+v", static, runtime, overlay, got, err, want) + } + combinations++ + } + } + } + } + } + if typedCombinations != 4*9*4*10 || combinations != 4*9*7*12*2+typedCombinations { + t.Fatalf("ran %d combinations, %d typed", combinations, typedCombinations) + } +} diff --git a/go/metrics.go b/go/metrics.go index baaa5b80..9ec04681 100644 --- a/go/metrics.go +++ b/go/metrics.go @@ -1,7 +1,6 @@ package dialcache import ( - "reflect" "strconv" "unicode/utf8" ) @@ -26,8 +25,7 @@ func (logger isolatedLogger) Debug(message string, details any) { logger.inner.Debug(message, details) } func (logger isolatedLogger) Warn(message string, details any) { - defer func() { _ = recover() }() - logger.inner.Warn(message, details) + warnDelivered(logger.inner, message, details) } func (logger isolatedLogger) Error(message string, details any) { defer func() { _ = recover() }() @@ -37,9 +35,25 @@ func FailureIsolatedLogger(logger Logger) Logger { if logger == nil { return nil } + if isolated, ok := logger.(isolatedLogger); ok { + return isolated + } return isolatedLogger{logger} } +// warnDelivered calls Warn in isolation and reports whether it returned +// without panicking; a Go logger returns nothing, so a panic is its only +// observable failure. +func warnDelivered(logger Logger, message string, details any) (delivered bool) { + if isolated, ok := logger.(isolatedLogger); ok { + logger = isolated.inner + } + // A panic skips the return, leaving delivered false. + defer func() { _ = recover() }() + logger.Warn(message, details) + return true +} + const ShadowLogKeyMaxBytes = 2048 const ShadowLogValueMaxBytes = 8192 const ShadowLogTruncationMarker = "...[truncated]" @@ -56,37 +70,12 @@ func clampLogUTF8(value string, maximum int) string { return value[:limit] + ShadowLogTruncationMarker } func PreviewShadowLogKey(value string) string { return clampLogUTF8(value, ShadowLogKeyMaxBytes) } -func PreviewShadowLogJSON(value any) (preview *string) { - defer func() { - if recover() != nil { - preview = nil - } - }() - if IsAbsent(value) { - return nil - } - encoded, err := appendJSON(nil, reflect.ValueOf(value), make(map[visit]bool)) - if err != nil { - return nil - } - text := clampLogUTF8(string(encoded), ShadowLogValueMaxBytes) - return &text -} - -type ShadowMismatchDetails struct { - CacheKey string `json:"cacheKey"` - CachedValueJSON *string `json:"cachedValueJson"` - SourceValueJSON *string `json:"sourceValueJson"` -} - -func ShadowMismatchLogDetails(key string, cached, source any) ShadowMismatchDetails { - return ShadowMismatchDetails{PreviewShadowLogKey(key), PreviewShadowLogJSON(cached), PreviewShadowLogJSON(source)} -} var metricKinds = map[string]string{ "shadowAge": "shadowValueAge", "recoveryAge": "staleRecoveryValueAge", "request": "request", "miss": "miss", "disabled": "disabled", "error": "error", "invalidation": "invalidation", "coalesced": "coalesced", "shadowValidation": "shadowValidation", "shadow": "shadowValidation", "shadowValueAge": "shadowValueAge", "observeShadowValueAge": "shadowValueAge", + "shadowMismatchLogging": "shadowMismatchLogging", "futureTimestampOffset": "futureTimestampOffset", "observeFutureTimestampOffset": "futureTimestampOffset", "futureOffset": "futureTimestampOffset", "staleRecovery": "staleRecovery", "recovery": "staleRecovery", "staleRecoveryValueAge": "staleRecoveryValueAge", "observeStaleRecoveryValueAge": "staleRecoveryValueAge", "compression": "compression", "get": "get", "observeGet": "get", "fallback": "fallback", "observeFallback": "fallback", @@ -111,7 +100,7 @@ func metricLabels(event Event, kind string) map[string]string { scope = event.Scope } labels["scope"] = scope - case "shadowValidation", "shadowValueAge", "staleRecovery", "staleRecoveryValueAge": + case "shadowValidation", "shadowValueAge", "shadowMismatchLogging", "staleRecovery", "staleRecoveryValueAge": outcome := eventString(event, "outcome") if outcome == "" { outcome = event.Outcome diff --git a/go/metrics_datadog.go b/go/metrics_datadog.go index 3ae693ea..5a57386c 100644 --- a/go/metrics_datadog.go +++ b/go/metrics_datadog.go @@ -27,12 +27,12 @@ type DatadogMetrics struct { var datadogSuffixes = map[string]string{ "request": "request.count", "miss": "miss.count", "disabled": "disabled.count", "error": "error.count", "invalidation": "invalidation.count", "coalesced": "coalesced.count", - "shadowValidation": "shadow.count", "shadowValueAge": "shadow.value_age", "futureTimestampOffset": "future_timestamp_offset", + "shadowValidation": "shadow.count", "shadowValueAge": "shadow.value_age", "shadowMismatchLogging": "shadow_mismatch_logging.count", "futureTimestampOffset": "future_timestamp_offset", "staleRecovery": "stale_recovery.count", "staleRecoveryValueAge": "stale_recovery.value_age", "compression": "compression.count", "get": "get.duration", "fallback": "fallback.duration", "serialization": "serialization.duration", "size": "serialization.size", "storedSize": "stored.size", "compressionRatio": "compression.ratio", "compressionDuration": "compression.duration", } -var counterKinds = map[string]bool{"request": true, "miss": true, "disabled": true, "error": true, "invalidation": true, "coalesced": true, "shadowValidation": true, "staleRecovery": true, "compression": true} +var counterKinds = map[string]bool{"request": true, "miss": true, "disabled": true, "error": true, "invalidation": true, "coalesced": true, "shadowValidation": true, "shadowMismatchLogging": true, "staleRecovery": true, "compression": true} var datadogNamespace = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z0-9_]+)*$`) func NewDatadogMetrics(options DatadogMetricsOptions) (*DatadogMetrics, error) { diff --git a/go/metrics_prometheus.go b/go/metrics_prometheus.go index eb14507a..22d896fa 100644 --- a/go/metrics_prometheus.go +++ b/go/metrics_prometheus.go @@ -26,6 +26,7 @@ func PrometheusCollectorSchemas(prefix string) []PrometheusCollectorSchema { {Kind: "coalesced", Name: prefix + "dialcache_coalesced_counter", Help: "DialCache requests coalesced onto in-flight work by sharing scope.", Labels: []string{"cache_namespace", "use_case", "key_type", "scope"}}, {Kind: "shadowValidation", Name: prefix + "dialcache_shadow_validation_counter", Help: "Sampled DialCache Redis shadow-validation outcomes.", Labels: []string{"cache_namespace", "use_case", "key_type", "outcome"}}, {Kind: "shadowValueAge", Name: prefix + "dialcache_shadow_value_age_histogram", Help: "Age in seconds of the validated Redis value at DialCache shadow verdict time.", Labels: []string{"cache_namespace", "use_case", "key_type", "outcome"}, Buckets: []float64{1, 5, 15, 60, 300, 900, 3600, 10800, 43200, 86400, 259200, 604800}}, + {Kind: "shadowMismatchLogging", Name: prefix + "dialcache_shadow_mismatch_logging_counter", Help: "DialCache confirmed shadow-mismatch warning outcomes.", Labels: []string{"cache_namespace", "use_case", "key_type", "outcome"}}, {Kind: "futureTimestampOffset", Name: prefix + "dialcache_future_timestamp_offset_histogram", Help: "Positive offset in seconds of Redis frames dated after the observing DialCache process clock.", Labels: []string{"cache_namespace", "use_case", "key_type", "layer"}, Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 5, 15, 60, 300, 900, 3600, 10800, 43200}}, {Kind: "staleRecovery", Name: prefix + "dialcache_stale_recovery_counter", Help: "DialCache stale-on-error Redis recovery outcomes.", Labels: []string{"cache_namespace", "use_case", "key_type", "outcome"}}, {Kind: "staleRecoveryValueAge", Name: prefix + "dialcache_stale_recovery_value_age_histogram", Help: "Age in seconds of Redis values served by DialCache stale-on-error recovery.", Labels: []string{"cache_namespace", "use_case", "key_type", "outcome"}, Buckets: []float64{1, 5, 15, 60, 300, 900, 3600, 10800, 43200, 86400, 259200, 604800}}, diff --git a/go/metrics_test.go b/go/metrics_test.go index f38a6a24..3c7347bf 100644 --- a/go/metrics_test.go +++ b/go/metrics_test.go @@ -48,7 +48,7 @@ func TestDatadogMetricNamesUnitsAndLabels(t *testing.T) { value float64 }{ {"request", "request.count", "increment", 1}, {"miss", "miss.count", "increment", 1}, {"disabled", "disabled.count", "increment", 1}, {"error", "error.count", "increment", 1}, {"invalidation", "invalidation.count", "increment", 1}, {"coalesced", "coalesced.count", "increment", 1}, - {"shadowValidation", "shadow.count", "increment", 1}, {"shadowAge", "shadow.value_age", "distribution", 0.25}, {"futureOffset", "future_timestamp_offset", "distribution", 0.25}, {"staleRecovery", "stale_recovery.count", "increment", 1}, {"recoveryAge", "stale_recovery.value_age", "distribution", 0.25}, + {"shadowValidation", "shadow.count", "increment", 1}, {"shadowAge", "shadow.value_age", "distribution", 0.25}, {"shadowMismatchLogging", "shadow_mismatch_logging.count", "increment", 1}, {"futureOffset", "future_timestamp_offset", "distribution", 0.25}, {"staleRecovery", "stale_recovery.count", "increment", 1}, {"recoveryAge", "stale_recovery.value_age", "distribution", 0.25}, {"compression", "compression.count", "increment", 1}, {"get", "get.duration", "distribution", 0.25}, {"fallback", "fallback.duration", "distribution", 0.25}, {"serialization", "serialization.duration", "distribution", 0.25}, {"size", "serialization.size", "distribution", 123}, {"storedSize", "stored.size", "distribution", 123}, {"compressionRatio", "compression.ratio", "distribution", 0.25}, {"compressionDuration", "compression.duration", "distribution", 0.25}, } for _, test := range cases { @@ -85,7 +85,15 @@ func TestDatadogMetricNamesUnitsAndLabels(t *testing.T) { if client.events[len(client.events)-1].method != "histogram" { t.Fatal("histogram option ignored") } - for _, namespace := range []string{"1bad", "has-dash", "two..dots", strings.Repeat("a", 201)} { + if mismatchLogging := client.events[8]; !reflect.DeepEqual(mismatchLogging.tags, map[string]string{"cache_namespace": "logical", "use_case": "lookup", "key_type": "item", "outcome": "mismatch"}) { + t.Fatalf("shadow mismatch logging tags: %+v", mismatchLogging) + } + // shadow_mismatch_logging.count is the longest suffix, so the namespace + // can be at most 170 characters. + if _, err := NewDatadogMetrics(DatadogMetricsOptions{Client: client, ObservationMetricType: "distribution", Namespace: strings.Repeat("a", 170)}); err != nil { + t.Fatal(err) + } + for _, namespace := range []string{"1bad", "has-dash", "two..dots", strings.Repeat("a", 171)} { if _, err := NewDatadogMetrics(DatadogMetricsOptions{Client: client, ObservationMetricType: "distribution", Namespace: namespace}); err == nil { t.Fatalf("accepted namespace %q", namespace) } @@ -106,7 +114,7 @@ func TestPrometheusWireSchemaMatchesTypeScriptBinding(t *testing.T) { } text := string(source) schemas := PrometheusCollectorSchemas("") - if len(schemas) != 19 { + if len(schemas) != 20 { t.Fatalf("collector count: %d", len(schemas)) } for _, schema := range schemas { @@ -232,7 +240,7 @@ func TestPrometheusExplicitBindingsPreserveExistingObservations(t *testing.T) { if err != nil { t.Fatal(err) } - if got := collectorDescriptions(registry); len(got) != 19 { + if got := collectorDescriptions(registry); len(got) != 20 { t.Fatalf("binding changed registry membership: %d", len(got)) } families, err := registry.Gather() @@ -322,28 +330,35 @@ func TestObserverIsolationAndBoundedNativeJSONLogs(t *testing.T) { logger.Debug("", nil) logger.Warn("", nil) logger.Error("", nil) + // Wrapping twice keeps one layer, so a Warn panic stays observable as a + // logger failure. + if FailureIsolatedLogger(logger) != logger || warnDelivered(FailureIsolatedLogger(logger), "", nil) || !warnDelivered(FailureIsolatedLogger(&boundaryLogger{}), "", nil) { + t.Fatal("logger isolation hid or invented a Warn failure") + } key := PreviewShadowLogKey(strings.Repeat("🙂", 1000)) if len(key) > ShadowLogKeyMaxBytes || !utf8.ValidString(key) || !strings.HasSuffix(key, ShadowLogTruncationMarker) { t.Fatalf("invalid key preview: bytes=%d", len(key)) } - value := PreviewShadowLogJSON(strings.Repeat("🙂", 3000)) - if value == nil || len(*value) > ShadowLogValueMaxBytes || !utf8.ValidString(*value) || !strings.HasSuffix(*value, ShadowLogTruncationMarker) { + preview := func(value any) any { + fields, _ := renderShadowMismatchLog(true, MismatchDiffOff, value, Absent, nil) + return fields["cachedValueJson"] + } + value, _ := preview(strings.Repeat("🙂", 3000)).(string) + if len(value) > ShadowLogValueMaxBytes || !utf8.ValidString(value) || !strings.HasSuffix(value, ShadowLogTruncationMarker) { t.Fatal("invalid value preview") } - if PreviewShadowLogJSON(Absent) != nil || PreviewShadowLogJSON(big.NewInt(1)) != nil { + if preview(Absent) != nil || preview(big.NewInt(1)) != nil { t.Fatal("unsupported values should have null previews") } cycle := map[string]any{} cycle["self"] = cycle - if PreviewShadowLogJSON(cycle) != nil { + if preview(cycle) != nil { t.Fatal("cyclic object should have null preview") } - plain := PreviewShadowLogJSON("") - if plain == nil || *plain != `""` { + if plain := preview(""); plain != `""` { t.Fatalf("native JSON was HTML escaped: %v", plain) } - null := PreviewShadowLogJSON(nil) - if null == nil || *null != "null" { + if null := preview(nil); null != "null" { t.Fatal("null confused with absent") } } diff --git a/go/policy.go b/go/policy.go index 4335eb0a..56dea636 100644 --- a/go/policy.go +++ b/go/policy.go @@ -41,8 +41,47 @@ type Policy struct { type ShadowPolicy struct { // Ramp is the shadow cohort percentage; nil and zero disable shadow work. Ramp *float64 - // LogMismatches emits one bounded warning per confirmed mismatch. + // LogMismatches is shorthand for the whole MismatchLogging group: true + // stands for Value true and Diff MismatchDiffValues, false for both off. + // Leaves set in this policy's MismatchLogging override what it expands to + // before a runtime overlay merges. LogMismatches *bool + // MismatchLogging selects the content of the one bounded warning emitted + // per confirmed mismatch. Omitted leaves default off. + MismatchLogging *MismatchLoggingPolicy +} + +// MismatchLoggingPolicy selects what a confirmed-mismatch warning carries. The +// warning is emitted when Value is true or Diff is above MismatchDiffOff. Nil +// leaves inherit, and runtime overlays merge each leaf independently. +type MismatchLoggingPolicy struct { + // Value includes bounded native JSON of both compared values. + Value *bool + // Diff includes the differing fields at this level. + Diff *MismatchDiffLevel +} + +// MismatchDiffLevel is how much of the structural difference between the +// compared values a confirmed-mismatch warning logs. +type MismatchDiffLevel string + +const ( + // MismatchDiffOff logs no difference. + MismatchDiffOff MismatchDiffLevel = "off" + // MismatchDiffPaths logs the RFC 6901 JSON Pointers of the differing fields. + MismatchDiffPaths MismatchDiffLevel = "paths" + // MismatchDiffValues logs those pointers with the cached and source values + // at each one. + MismatchDiffValues MismatchDiffLevel = "values" +) + +func (level MismatchDiffLevel) valid() bool { + return level == MismatchDiffOff || level == MismatchDiffPaths || level == MismatchDiffValues +} + +// on reports whether the level logs any difference. +func (level MismatchDiffLevel) on() bool { + return level == MismatchDiffPaths || level == MismatchDiffValues } // PolicyProvider returns a sparse runtime overlay for one invocation. A nil @@ -136,16 +175,31 @@ func (p *PolicyOverlay) runtimePolicy() (any, error) { m["remoteReadTimeoutMs"] = wholeUnits(*p.RemoteReadTimeout, time.Millisecond) } if p.Shadow != nil { - shadow := map[string]any{} - if p.Shadow.Ramp != nil { - shadow["ramp"] = *p.Shadow.Ramp + m["shadow"] = shadowPolicyMap(p.Shadow) + } + return m, nil +} + +// shadowPolicyMap writes a typed shadow group in the shared JSON shape. +func shadowPolicyMap(p *ShadowPolicy) map[string]any { + shadow := map[string]any{} + if p.Ramp != nil { + shadow["ramp"] = *p.Ramp + } + if p.LogMismatches != nil { + shadow["logMismatches"] = *p.LogMismatches + } + if group := p.MismatchLogging; group != nil { + leaves := map[string]any{} + if group.Value != nil { + leaves["value"] = *group.Value } - if p.Shadow.LogMismatches != nil { - shadow["logMismatches"] = *p.Shadow.LogMismatches + if group.Diff != nil { + leaves["diff"] = string(*group.Diff) } - m["shadow"] = shadow + shadow["mismatchLogging"] = leaves } - return m, nil + return shadow } type PolicyDefaults struct{ RemoteReadTimeout time.Duration } @@ -157,12 +211,19 @@ type ResolvedLayer struct { Ramp float64 } type ResolvedShadow struct { - Enabled bool // cohort selection only; admission still needs an eligible path and hook - Ramp float64 - LogMismatches bool - ConfigError bool - LoggingConfigError bool // record only if a job is admitted, as in TypeScript + Enabled bool // cohort selection only; admission still needs an eligible path and hook + Ramp float64 + // LogValue and LogDiff are the effective mismatch-logging leaves: each + // layer's LogMismatches expands first, then the leaves merge, and an + // invalid leaf acts off. + LogValue bool + LogDiff MismatchDiffLevel + ConfigError bool + // LoggingConfigError covers every invalid logging leaf and shorthand; record + // it once, only if a job is admitted, as in TypeScript. + LoggingConfigError bool } + type ResolvedPolicy struct { RequestLocal bool Coalesce bool @@ -174,6 +235,9 @@ type ResolvedPolicy struct { Shadow ResolvedShadow } +// logsMismatches reports whether a confirmed mismatch emits a warning. +func (s ResolvedShadow) logsMismatches() bool { return s.LogValue || s.LogDiff.on() } + func policyNumber(value any) (float64, bool) { if number, ok := value.(json.Number); ok { n, err := number.Float64() @@ -325,6 +389,27 @@ func ParsePolicy(config any) (Policy, error) { } p.Shadow.LogMismatches = &flag } + if v, present := optionalLeaf(shadow, "mismatchLogging"); present { + group, err := policyMap(v, "shadow.mismatchLogging") + if err != nil { + return p, err + } + p.Shadow.MismatchLogging = &MismatchLoggingPolicy{} + if v, present := optionalLeaf(group, "value"); present { + flag, ok := v.(bool) + if !ok { + return p, invalidPolicy("shadow.mismatchLogging.value must be boolean") + } + p.Shadow.MismatchLogging.Value = &flag + } + if v, present := optionalLeaf(group, "diff"); present { + level, ok := resolveDiffLeaf(v) + if !ok { + return p, invalidPolicy(`shadow.mismatchLogging.diff must be "off", "paths" or "values"`) + } + p.Shadow.MismatchLogging.Diff = &level + } + } } return p, ValidatePolicy(p) } @@ -357,6 +442,9 @@ func ValidatePolicy(p Policy) error { return invalidPolicy("static shadow ramp must be between zero and 100") } } + if p.Shadow != nil && p.Shadow.MismatchLogging != nil && p.Shadow.MismatchLogging.Diff != nil && !p.Shadow.MismatchLogging.Diff.valid() { + return invalidPolicy(`static shadow mismatch diff must be "off", "paths" or "values"`) + } return nil } @@ -390,6 +478,15 @@ func SnapshotPolicy(p Policy) Policy { copy := *p.Shadow copy.Ramp = cloneFloat(copy.Ramp) copy.LogMismatches = cloneBool(copy.LogMismatches) + if copy.MismatchLogging != nil { + group := *copy.MismatchLogging + group.Value = cloneBool(group.Value) + if group.Diff != nil { + diff := *group.Diff + group.Diff = &diff + } + copy.MismatchLogging = &group + } p.Shadow = © } return p @@ -421,18 +518,104 @@ func staticPolicyMap(p Policy) map[string]any { m["remoteReadTimeoutMs"] = int64(*p.RemoteReadTimeout / time.Millisecond) } if p.Shadow != nil { - shadow := map[string]any{} - if p.Shadow.Ramp != nil { - shadow["ramp"] = *p.Shadow.Ramp - } - if p.Shadow.LogMismatches != nil { - shadow["logMismatches"] = *p.Shadow.LogMismatches - } - m["shadow"] = shadow + m["shadow"] = shadowPolicyMap(p.Shadow) } return m } +// mismatchLeaves are the mismatch-logging leaves one shadow group writes, +// before validation. Absent marks an unwritten leaf; invalid marks a malformed +// shorthand, which still owes one config_resolution. +type mismatchLeaves struct { + value, diff any + invalid bool +} + +// expandMismatchLogging expands one shadow group's logMismatches shorthand +// into the leaves it stands for, true as Value true and Diff "values" and +// false as both off, then lets the leaves written in the same group's +// mismatchLogging override them. A malformed shorthand expands to both leaves +// off and is marked invalid. A mismatchLogging that is not an object fails +// resolution like a malformed layer map. +func expandMismatchLogging(shadow map[string]any) (mismatchLeaves, error) { + leaves := mismatchLeaves{value: Absent, diff: Absent} + if shorthand, present := optionalLeaf(shadow, "logMismatches"); present { + switch shorthand { + case true: + leaves.value, leaves.diff = true, MismatchDiffValues + case false: + leaves.value, leaves.diff = false, MismatchDiffOff + default: + leaves.value, leaves.diff, leaves.invalid = false, MismatchDiffOff, true + } + } + v, present := optionalLeaf(shadow, "mismatchLogging") + if !present { + return leaves, nil + } + group, err := policyMap(v, "shadow.mismatchLogging") + if err != nil { + return leaves, err + } + if v, present := optionalLeaf(group, "value"); present { + leaves.value = v + } + if v, present := optionalLeaf(group, "diff"); present { + leaves.diff = v + } + return leaves, nil +} + +// overlaidBy lets each leaf a runtime group wrote win over the inherited +// static one, whose typed shorthand is never invalid. +func (inherited mismatchLeaves) overlaidBy(written mismatchLeaves) mismatchLeaves { + if IsAbsent(written.value) { + written.value = inherited.value + } + if IsAbsent(written.diff) { + written.diff = inherited.diff + } + return written +} + +// resolve validates each leaf independently: an omitted leaf is off, an +// invalid leaf acts off, and invalid reports that the group owes one +// config_resolution. +func (leaves mismatchLeaves) resolve() (value bool, diff MismatchDiffLevel, invalid bool) { + value, validValue := resolveValueLeaf(leaves.value) + diff, validDiff := resolveDiffLeaf(leaves.diff) + return value, diff, leaves.invalid || !validValue || !validDiff +} + +// resolveValueLeaf returns the leaf and whether it is valid; an invalid leaf +// acts off. +func resolveValueLeaf(leaf any) (bool, bool) { + if IsAbsent(leaf) { + return false, true + } + value, valid := leaf.(bool) + return value, valid +} + +// resolveDiffLeaf accepts a level as a string or a MismatchDiffLevel and +// returns it and whether it is valid; an invalid leaf acts off. +func resolveDiffLeaf(leaf any) (MismatchDiffLevel, bool) { + if IsAbsent(leaf) { + return MismatchDiffOff, true + } + var level MismatchDiffLevel + switch v := leaf.(type) { + case string: + level = MismatchDiffLevel(v) + case MismatchDiffLevel: + level = v + } + if !level.valid() { + return MismatchDiffOff, false + } + return level, true +} + // ResolvePolicy merges sparse leaves once. A malformed container, boolean or // read deadline is an invocation-wide error; TTL/ramp errors disable only that // layer, while an invalid recovery option preserves valid remote serving. @@ -448,6 +631,9 @@ func ResolvePolicy(base Policy, overlay RuntimePolicy, identity Identity, defaul return resolved, err } merged := staticPolicyMap(base) + staticShadow, _ := merged["shadow"].(map[string]any) + // The static group is written from typed fields, so it always expands. + logging, _ := expandMismatchLogging(staticShadow) var raw any if overlay != nil { var err error @@ -478,7 +664,15 @@ func ResolvePolicy(base Policy, overlay RuntimePolicy, identity Identity, defaul } leaves := []string{"local", "remote"} if name == "shadow" { - leaves = []string{"ramp", "logMismatches"} + leaves = []string{"ramp"} + // Each layer's shorthand expands before the leaf-wise merge, so a + // runtime logMismatches replaces both inherited leaves while a + // leaf written beside it still wins within its own layer. + written, err := expandMismatchLogging(incoming) + if err != nil { + return resolved, err + } + logging = logging.overlaidBy(written) } for _, leaf := range leaves { if v, present := optionalLeaf(incoming, leaf); present { @@ -566,11 +760,7 @@ func ResolvePolicy(base Policy, overlay RuntimePolicy, identity Identity, defaul resolved.Shadow.Enabled = ramp >= 100 || (ramp > 0 && Cohort(logical, "shadow") < ramp) } } - if v, present := optionalLeaf(shadow, "logMismatches"); present { - flag, valid := v.(bool) - resolved.Shadow.LogMismatches = flag - resolved.Shadow.LoggingConfigError = !valid - } } + resolved.Shadow.LogValue, resolved.Shadow.LogDiff, resolved.Shadow.LoggingConfigError = logging.resolve() return resolved, nil } diff --git a/go/protocol_test.go b/go/protocol_test.go index 4910043b..4ed40500 100644 --- a/go/protocol_test.go +++ b/go/protocol_test.go @@ -479,6 +479,15 @@ func TestProtocolRemainingVectors(t *testing.T) { } }) } + // W10: the built-in walk over two JSON values and the warning fields + // derived from it. + var mismatches []mismatchPathVector + if err := json.Unmarshal(groups["mismatchPathVectors"], &mismatches); err != nil { + t.Fatal(err) + } + for _, v := range mismatches { + t.Run(v.Name, func(t *testing.T) { checkMismatchPathVector(t, v) }) + } count := 0 for group, raw := range groups { if group == "schemaVersion" { @@ -493,7 +502,7 @@ func TestProtocolRemainingVectors(t *testing.T) { selection := os.Getenv("DIALCACHE_PROTOCOL_CORPUS") expected := 0 if selection != "generated" { - expected = 134 + expected = 191 } if selection != "fixed" { for _, rows := range generatedProtocolGroups(t) { diff --git a/go/replay_coordinator_test.go b/go/replay_coordinator_test.go index d7a81435..474ee38b 100644 --- a/go/replay_coordinator_test.go +++ b/go/replay_coordinator_test.go @@ -831,6 +831,77 @@ func TestReplayTransportValidatesObservationsLocally(t *testing.T) { }) } +// The shadow profile's warning records and inputs cross the transport: the +// full mismatch warning with its paths as one JSON string, the +// shadowMismatchLogging event, a call that returned the unrenderable value, +// the unrenderable resolve and the logger fault. Their malformed neighbours +// are driver or coordinator defects the local schema check rejects. +func TestReplayTransportCarriesShadowWarningRecords(t *testing.T) { + schema, err := readReplaySchema() + if err != nil { + t.Fatal(err) + } + definitions := bm(schema["$defs"]) + // The driver's encoding of a warning the library logs, failed fields and all. + warning := behaviorWarningObservation(obj{"cacheNamespace": "urn", "useCase": "Behavior", "keyType": "id", "outcome": "mismatch", + "cacheKey": "urn:id:1#Behavior", "cachedValueAgeSeconds": 0.5, "mismatchPaths": []string{"", "/a= c.settings.shadowCapacity { c.mu.Unlock() - x.shadowEvent(shadowVerdict{outcome: "dropped"}) + x.shadowEvent(shadowVerdict[T]{outcome: "dropped"}) return } f := &shadowFlight{stop: make(chan struct{})} @@ -58,7 +58,7 @@ func (x *execution[T]) scheduleShadow(frame *Frame, source *pending[T], started deferWork(c.settings.clock, func() { x.runShadow(f, frame, source, started) }) } -func (x *execution[T]) shadowEvent(v shadowVerdict) { +func (x *execution[T]) shadowEvent(v shadowVerdict[T]) { d := x.labels("") d["outcome"] = v.outcome e := Event{Kind: "shadowValidation", Key: x.key, Data: d, Outcome: v.outcome} @@ -73,31 +73,61 @@ func (x *execution[T]) shadowEvent(v shadowVerdict) { } x.event("shadowAge", "", map[string]any{"outcome": v.outcome, "seconds": float64(age) / 1000}) } - if v.outcome == "mismatch" && x.policy.Shadow.LogMismatches && x.cache.settings.logger != nil { - func() { - defer func() { recover() }() - details := x.labels("") - details["outcome"] = "mismatch" + if v.outcome == "mismatch" && x.policy.Shadow.logsMismatches() { + x.warnShadowMismatch(v) + } +} - preview := ShadowMismatchLogDetails(x.key, v.cached, v.source) - details["cacheKey"] = preview.CacheKey - if preview.CachedValueJSON != nil { - details["cachedValueJson"] = *preview.CachedValueJSON - } else { - details["cachedValueJson"] = nil - } - if preview.SourceValueJSON != nil { - details["sourceValueJson"] = *preview.SourceValueJSON - } else { - details["sourceValueJson"] = nil - } - x.cache.settings.logger.Warn("DialCache shadow validation mismatch", details) - }() +// warnShadowMismatch emits the confirmed-mismatch warning and its one +// shadowMismatchLogging event. It runs after the verdict was recorded and +// never changes it; a construction failure logs less, never more, and nothing +// reaches the caller. +func (x *execution[T]) warnShadowMismatch(v shadowVerdict[T]) { + shadow := x.policy.Shadow + age := *v.age + if age < 0 { + age = 0 + } + details := x.labels("") + details["outcome"] = "mismatch" + details["cachedValueAgeSeconds"] = float64(age) / 1000 + var fields map[string]any + failure := shadowLogRenderError + func() { + // Construction is fail-closed by design; keep the metadata warning if + // anything unforeseen still panics while rendering it. + defer func() { _ = recover() }() + details["cacheKey"] = PreviewShadowLogKey(x.key) + var hook func() ([]MismatchEntry, error) + if x.op.MismatchDiff != nil { + hook = func() ([]MismatchEntry, error) { return x.op.MismatchDiff(v.cached, v.source) } + } + fields, failure = renderShadowMismatchLog(shadow.LogValue, shadow.LogDiff, v.cached, v.source, hook) + }() + for name, field := range fields { + details[name] = field + } + if failure != "" { + details["loggingOutcome"] = failure + } + delivered := warnDelivered(x.cache.settings.logger, "DialCache shadow validation mismatch", details) + // hook_error ranks above render_error, which ranks above logger_error. + outcome := failure + if outcome == "" { + outcome = "logged" + if !delivered { + outcome = "logger_error" + } } + x.event("shadowMismatchLogging", "", map[string]any{"outcome": outcome}) } func (x *execution[T]) runShadow(f *shadowFlight, frame *Frame, source *pending[T], started time.Duration) { clock := x.cache.settings.clock + // The verdict and its mismatch warning keep the slot too, so capacity + // bounds the hook and the built-in walk as it bounds the comparator. + reported := make(chan struct{}) + defer close(reported) if source == nil { started = elapsedNow(clock) } @@ -141,7 +171,7 @@ func (x *execution[T]) runShadow(f *shadowFlight, frame *Frame, source *pending[ x.duration("get", "remote_shadow", begin, nil) return r, err } - validation := startPending(func() (shadowVerdict, error) { + validation := startPending(func() (shadowVerdict[T], error) { defer func() { // Reads keep the slot after a read timeout; owned source/codec/write // work already keeps this operation blocked until its raw completion. @@ -149,6 +179,7 @@ func (x *execution[T]) runShadow(f *shadowFlight, frame *Frame, source *pending[ for _, r := range reads { <-r.done } + <-reported x.cache.mu.Lock() if x.cache.shadows[x.key] == f { delete(x.cache.shadows, x.key) @@ -156,7 +187,7 @@ func (x *execution[T]) runShadow(f *shadowFlight, frame *Frame, source *pending[ x.cache.mu.Unlock() }() }() - out := func(name string) (shadowVerdict, error) { return shadowVerdict{outcome: name}, nil } + out := func(name string) (shadowVerdict[T], error) { return shadowVerdict[T]{outcome: name}, nil } if expired() { return out("timeout") } @@ -243,7 +274,7 @@ func (x *execution[T]) runShadow(f *shadowFlight, frame *Frame, source *pending[ } age := clock.WallMS() - int64(frame.CreatedAtMS) if matches { - return shadowVerdict{outcome: "match", age: &age}, nil + return shadowVerdict[T]{outcome: "match", age: &age}, nil } confirmation, err := read(false, true) if err != nil { @@ -256,13 +287,13 @@ func (x *execution[T]) runShadow(f *shadowFlight, frame *Frame, source *pending[ return out("superseded") } age = clock.WallMS() - int64(frame.CreatedAtMS) - return shadowVerdict{outcome: "mismatch", age: &age, cached: cached, source: value}, nil + return shadowVerdict[T]{outcome: "mismatch", age: &age, cached: cached, source: value}, nil }) verdict, err := awaitDeadline(clock, validation, started, budget, func() error { return &FallbackTimeoutError{UseCase: x.op.Identity.UseCase, Timeout: time.Duration(budget) * time.Millisecond} }, f.abandon) if err != nil { - verdict = shadowVerdict{outcome: "timeout"} + verdict = shadowVerdict[T]{outcome: "timeout"} } x.shadowEvent(verdict) } diff --git a/go/shadow_log.go b/go/shadow_log.go new file mode 100644 index 00000000..43e8d62c --- /dev/null +++ b/go/shadow_log.go @@ -0,0 +1,319 @@ +package dialcache + +import ( + "bytes" + "encoding/json" + "reflect" + "sort" + "strconv" + "strings" + "unicode/utf8" +) + +// Bounds of the confirmed-mismatch difference fields. +const ( + shadowLogDiffMaxBytes = 8192 + shadowMismatchMaxEntries = 64 + shadowMismatchPathMaxBytes = 512 + shadowMismatchSideMaxBytes = 1024 + // shadowMismatchMaxDepth counts the reference tokens in the deepest pointer + // the built-in walk reports. + shadowMismatchMaxDepth = 32 + // shadowMismatchPointerCopyBytes bounds the pointer the walk copies into a + // kept entry. A longer pointer still exceeds shadowMismatchPathMaxBytes once + // cut there, even when the cut splits a character, so it clamps to the same + // text as the whole pointer. + shadowMismatchPointerCopyBytes = shadowMismatchPathMaxBytes + utf8.UTFMax +) + +// Why warning construction failed: the difference computation (hook or +// built-in walk) ranks above JSON rendering. +const ( + shadowLogHookError = "hook_error" + shadowLogRenderError = "render_error" +) + +// renderShadowMismatchLog builds the content fields the effective leaves +// request for one confirmed mismatch, and the failure that nulled any of them, +// or "" when every requested field rendered. Each root is rendered to native +// JSON at most once and every built-in field derives from that rendering, so a +// stateful MarshalJSON cannot put into one field what another omits. Every +// failure fails its fields closed to nil, and a failed hook never falls back to +// the built-in computation. An Absent root, which a negative-cache entry +// stores, is not a failure: its value field is nil, and the built-in reports +// the other root whole at "". hook is nil when the operation has none. +func renderShadowMismatchLog(logValue bool, diff MismatchDiffLevel, cached, source any, hook func() ([]MismatchEntry, error)) (fields map[string]any, failure string) { + fields = map[string]any{} + var cachedJSON, sourceJSON []byte + renderFailed, diffFailed := false, false + if logValue || (diff.on() && hook == nil) { + var cachedFailed, sourceFailed bool + cachedJSON, cachedFailed = renderRoot(cached) + sourceJSON, sourceFailed = renderRoot(source) + renderFailed = cachedFailed || sourceFailed + } + if diff.on() { + withValues := diff == MismatchDiffValues + var entries []MismatchEntry + count, computed := 0, false + if hook != nil { + entries, count, computed = runMismatchHook(hook) + diffFailed = !computed + } else if !renderFailed { + entries, count, computed = walkRenderedJSON(cachedJSON, sourceJSON) + diffFailed = !computed + } + if !computed { + fields["mismatchPaths"], fields["mismatchPathCount"] = nil, nil + if withValues { + fields["diffJson"] = nil + } + } else { + // "paths" never renders a side. + paths := make([]string, len(entries)) + for i, entry := range entries { + paths[i] = clampLogUTF8(entry.Path, shadowMismatchPathMaxBytes) + } + fields["mismatchPaths"], fields["mismatchPathCount"] = paths, count + if withValues { + if encoded, rendered := appendMismatchEntries(nil, entries); rendered { + fields["diffJson"] = clampLogUTF8(string(encoded), shadowLogDiffMaxBytes) + } else { + fields["diffJson"] = nil + renderFailed = true + } + } + } + } + if logValue { + fields["cachedValueJson"], fields["sourceValueJson"] = valueLogField(cachedJSON), valueLogField(sourceJSON) + } + switch { + case diffFailed: + failure = shadowLogHookError + case renderFailed: + failure = shadowLogRenderError + } + return fields, failure +} + +// renderJSON renders one value to native JSON, honoring json.Marshaler at +// every depth when marshalers is set. A value with no rendering, including one +// whose MarshalJSON panics, reports false. +func renderJSON(value any, marshalers bool) (encoded []byte, rendered bool) { + // A panic skips the return, leaving both results zero. + defer func() { _ = recover() }() + encoded, err := appendJSON(nil, reflect.ValueOf(value), make(map[visit]bool), marshalers) + return encoded, err == nil +} + +// renderRoot renders a compared value; nil bytes mean no rendering, and failed +// separates a value with no rendering from an Absent one. +func renderRoot(value any) (encoded []byte, failed bool) { + if IsAbsent(value) { + return nil, false + } + encoded, rendered := renderJSON(value, false) + return encoded, !rendered +} + +func valueLogField(encoded []byte) any { + if encoded == nil { + return nil + } + return clampLogUTF8(string(encoded), ShadowLogValueMaxBytes) +} + +// walkRenderedJSON runs the built-in computation over two root renderings; +// nil is an Absent root, since a root that failed to render never reaches it. +// computed is false when a rendering does not parse back, as for nesting past +// encoding/json's depth limit. +func walkRenderedJSON(cachedJSON, sourceJSON []byte) (entries []MismatchEntry, count int, computed bool) { + if bytes.Equal(cachedJSON, sourceJSON) { + return nil, 0, true + } + cached, source := any(Absent), any(Absent) + for _, side := range []struct { + encoded []byte + parsed *any + }{{cachedJSON, &cached}, {sourceJSON, &source}} { + if side.encoded == nil { + continue + } + var parsed any + if err := json.Unmarshal(side.encoded, &parsed); err != nil { + return nil, 0, false + } + *side.parsed = parsed + } + if IsAbsent(cached) || IsAbsent(source) { + return []MismatchEntry{{Path: "", Cached: cached, Source: source}}, 1, true + } + entries, count = walkJSON(cached, source) + return entries, count, true +} + +type jsonKind int + +const ( + jsonPrimitive jsonKind = iota + jsonArray + jsonObject +) + +func kindOfJSON(value any) jsonKind { + switch value.(type) { + case []any: + return jsonArray + case map[string]any: + return jsonObject + } + return jsonPrimitive +} + +// walkJSON is the built-in difference computation over two parsed JSON +// values. Objects recurse over the union of their member names in UTF-16 +// code-unit order, and a member present on one side only is reported at its +// pointer with the other side Absent. Arrays of equal length recurse by index; +// arrays of different length are reported once at their own pointer. Any other +// differing pair, roots of different kinds included, is reported at its pointer +// with both sides, and a differing pair of containers at shadowMismatchMaxDepth +// reference tokens is reported whole without descending. Entries keep walk +// order and are cut at shadowMismatchMaxEntries; count is the uncut total. +func walkJSON(cached, source any) (entries []MismatchEntry, count int) { + entries = []MismatchEntry{} + // pointer holds the escaped reference tokens down to the pair being walked: + // each container appends a child's token and cuts it off again. Only a + // retained entry copies it into a string, so a member that compares equal or + // falls past the cut never copies its ancestors' tokens, and a deep chain of + // long names keeps the walk linear in the renderings. The copy stops at + // shadowMismatchPointerCopyBytes: the warning clamps every pointer to + // shadowMismatchPathMaxBytes, which keeps only bytes inside that prefix. + var pointer []byte + report := func(cachedSide, sourceSide any) { + count++ + if len(entries) < shadowMismatchMaxEntries { + kept := pointer[:min(len(pointer), shadowMismatchPointerCopyBytes)] + entries = append(entries, MismatchEntry{Path: string(kept), Cached: cachedSide, Source: sourceSide}) + } + } + var walk func(left, right any, depth int) + walk = func(left, right any, depth int) { + kind := kindOfJSON(left) + if kind != kindOfJSON(right) { + report(left, right) + return + } + if kind == jsonPrimitive { + if left != right { + report(left, right) + } + return + } + if depth >= shadowMismatchMaxDepth { + if !reflect.DeepEqual(left, right) { + report(left, right) + } + return + } + parent := len(pointer) + if kind == jsonArray { + leftItems, rightItems := left.([]any), right.([]any) + if len(leftItems) != len(rightItems) { + report(left, right) + return + } + for index := range leftItems { + pointer = strconv.AppendInt(append(pointer[:parent], '/'), int64(index), 10) + walk(leftItems[index], rightItems[index], depth+1) + } + pointer = pointer[:parent] + return + } + leftMembers, rightMembers := left.(map[string]any), right.(map[string]any) + names := make([]string, 0, len(leftMembers)+len(rightMembers)) + for name := range leftMembers { + names = append(names, name) + } + for name := range rightMembers { + if _, shared := leftMembers[name]; !shared { + names = append(names, name) + } + } + sort.Slice(names, func(i, j int) bool { return utf16Less(names[i], names[j]) }) + for _, name := range names { + pointer = append(append(pointer[:parent], '/'), escapePointerToken(name)...) + leftMember, inLeft := leftMembers[name] + rightMember, inRight := rightMembers[name] + switch { + case !inRight: + report(leftMember, Absent) + case !inLeft: + report(Absent, rightMember) + default: + walk(leftMember, rightMember, depth+1) + } + } + pointer = pointer[:parent] + } + walk(cached, source, 0) + return entries, count +} + +func escapePointerToken(name string) string { + return strings.ReplaceAll(strings.ReplaceAll(name, "~", "~0"), "/", "~1") +} + +// runMismatchHook runs the operation's hook, which replaces the built-in +// computation. A returned error or panic fails it; the entries are cut at +// shadowMismatchMaxEntries and count is the uncut total. +func runMismatchHook(hook func() ([]MismatchEntry, error)) (entries []MismatchEntry, count int, computed bool) { + entries, err := callSafely(hook) + if err != nil { + return nil, 0, false + } + count = len(entries) + if count > shadowMismatchMaxEntries { + entries = entries[:shadowMismatchMaxEntries] + } + return entries, count, true +} + +// appendMismatchEntries renders entries with their bounded sides, the one +// clamp rule for built-in and hook entries: each pointer is clamped, and each +// side that is not Absent is rendered once and kept as JSON, or as its clamped +// text when the rendering exceeds the side cap. A side honors json.Marshaler at +// every depth, which only a hook's sides can hold, so a redacting MarshalJSON +// keeps its secret out of diffJson; the roots keep JSONCodec's encoding. +// rendered is false at the first side with no JSON rendering, and later sides +// are not rendered. +func appendMismatchEntries(dst []byte, entries []MismatchEntry) (encoded []byte, rendered bool) { + dst = append(dst, '[') + for index, entry := range entries { + if index > 0 { + dst = append(dst, ',') + } + dst = append(dst, `{"path":`...) + dst = quoteJSON(dst, clampLogUTF8(entry.Path, shadowMismatchPathMaxBytes)) + for _, side := range [2]struct { + name string + value any + }{{"cached", entry.Cached}, {"source", entry.Source}} { + if IsAbsent(side.value) { + continue + } + rendering, ok := renderJSON(side.value, true) + if !ok { + return nil, false + } + dst = append(dst, `,"`+side.name+`":`...) + if len(rendering) > shadowMismatchSideMaxBytes { + dst = quoteJSON(dst, clampLogUTF8(string(rendering), shadowMismatchSideMaxBytes)) + } else { + dst = append(dst, rendering...) + } + } + dst = append(dst, '}') + } + return append(dst, ']'), true +} diff --git a/go/shadow_log_test.go b/go/shadow_log_test.go new file mode 100644 index 00000000..9ff0edc3 --- /dev/null +++ b/go/shadow_log_test.go @@ -0,0 +1,679 @@ +package dialcache + +import ( + "encoding/json" + "errors" + "fmt" + "math/big" + "os" + "reflect" + "runtime" + "strings" + "testing" + "time" +) + +// mismatchPathVector is one row of the mismatchPathVectors group in +// formal/catalogs/protocol-vectors.json: the built-in walk's hand-authored +// expectations, which every port replays and which the documented examples in +// docs/shadow-validation.md come from. +type mismatchPathVector struct { + Name string `json:"name"` + Cached, Source json.RawMessage // nil when the row omits the side + Entries json.RawMessage `json:"entries"` + Count int `json:"count"` +} + +// sharedMismatchPathVectors reads the group whatever DIALCACHE_PROTOCOL_CORPUS +// selects, so the ordinary suite pins the walk as the protocol suite does. +func sharedMismatchPathVectors(t *testing.T) []mismatchPathVector { + t.Helper() + raw, err := os.ReadFile("../formal/catalogs/protocol-vectors.json") + if err != nil { + t.Fatal(err) + } + var corpus struct { + Vectors []mismatchPathVector `json:"mismatchPathVectors"` + } + if err := json.Unmarshal(raw, &corpus); err != nil { + t.Fatal(err) + } + if len(corpus.Vectors) == 0 { + t.Fatal("protocol-vectors.json has no mismatchPathVectors") + } + return corpus.Vectors +} + +// mismatchVectorSide is a vector side as the warning receives it: Absent when +// the row omits it (D13), else its parsed JSON. +func mismatchVectorSide(t *testing.T, raw json.RawMessage) any { + t.Helper() + if raw == nil { + return Absent + } + return parseJSONValue(t, string(raw)) +} + +func parseJSONValue(t *testing.T, text string) any { + t.Helper() + var value any + if err := json.Unmarshal([]byte(text), &value); err != nil { + t.Fatal(err) + } + return value +} + +// boundedEntries renders walk entries with the diffJson clamp rule, unclamped +// as a whole, and parses them back for comparison as JSON. +func boundedEntries(t *testing.T, entries []MismatchEntry) []any { + t.Helper() + encoded, rendered := appendMismatchEntries(nil, entries) + if !rendered { + t.Fatal("walk entries have no rendering") + } + parsed, _ := parseJSONValue(t, string(encoded)).([]any) + return parsed +} + +func entryPaths(entries []any) []string { + paths := []string{} + for _, entry := range entries { + paths = append(paths, entry.(map[string]any)["path"].(string)) + } + return paths +} + +func renderMismatch(value bool, diff MismatchDiffLevel, cached, source any) (map[string]any, string) { + return renderShadowMismatchLog(value, diff, cached, source, nil) +} + +func expectLog(t *testing.T, fields map[string]any, failure string, wantFields map[string]any, wantFailure string) { + t.Helper() + if failure != wantFailure || !reflect.DeepEqual(fields, wantFields) { + t.Fatalf("rendered %#v with failure %q, want %#v with %q", fields, failure, wantFields, wantFailure) + } +} + +func nestMembers(depth int, leaf any) any { + value := leaf + for level := 0; level < depth; level++ { + value = map[string]any{"n": value} + } + return value +} + +// checkMismatchPathVector replays one row: the walk, its entries compared as +// parsed JSON, and both projections of one entry list in the warning fields, +// the paths and count at diff "paths" and the entries with their values at +// "values". An omitted side is Absent (D13) and reaches the walk only through +// rendering. +func checkMismatchPathVector(t *testing.T, vector mismatchPathVector) { + t.Helper() + cached, source := mismatchVectorSide(t, vector.Cached), mismatchVectorSide(t, vector.Source) + entries, ok := parseJSONValue(t, string(vector.Entries)).([]any) + if !ok { + t.Fatal("vector entries are not an array") + } + if !IsAbsent(cached) && !IsAbsent(source) { + walked, count := walkJSON(cached, source) + if count != vector.Count || !reflect.DeepEqual(boundedEntries(t, walked), entries) { + t.Fatalf("walk: %d %s, want %d %s", count, bjson(boundedEntries(t, walked)), vector.Count, vector.Entries) + } + } + paths := entryPaths(entries) + values, failure := renderMismatch(false, MismatchDiffValues, cached, source) + diffJSON, rendered := values["diffJson"].(string) + if failure != "" || !rendered || len(values) != 3 || values["mismatchPathCount"] != vector.Count || + !reflect.DeepEqual(values["mismatchPaths"], paths) || !reflect.DeepEqual(parseJSONValue(t, diffJSON), entries) { + t.Fatalf("diff values fields: %#v %q", values, failure) + } + fields, failure := renderMismatch(false, MismatchDiffPaths, cached, source) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": paths, "mismatchPathCount": vector.Count}, "") +} + +func TestBuiltInMismatchPathVectors(t *testing.T) { + for _, vector := range sharedMismatchPathVectors(t) { + t.Run(vector.Name, func(t *testing.T) { checkMismatchPathVector(t, vector) }) + } +} + +func TestBuiltInMismatchWalkReadsOwnMembersOnly(t *testing.T) { + // Names that are prototype members in JavaScript are ordinary members. + cached := parseJSONValue(t, `{"constructor":1,"__proto__":{"a":1}}`) + source := parseJSONValue(t, `{"toString":"x","valueOf":null,"__proto__":{"a":2}}`) + want := parseJSONValue(t, `[{"path":"/__proto__/a","cached":1,"source":2},{"path":"/constructor","cached":1},{"path":"/toString","source":"x"},{"path":"/valueOf","source":null}]`) + entries, count := walkJSON(cached, source) + if count != 4 || !reflect.DeepEqual(boundedEntries(t, entries), want) { + t.Fatalf("own-member walk: %d %#v", count, boundedEntries(t, entries)) + } + nest := func(leaf string) any { + return parseJSONValue(t, strings.Repeat(`{"a":`, 32)+leaf+strings.Repeat("}", 32)) + } + entries, count = walkJSON(nest(`{"__proto__":{}}`), nest(`{"b":{}}`)) + want = parseJSONValue(t, `[{"path":"`+strings.Repeat("/a", 32)+`","cached":{"__proto__":{}},"source":{"b":{}}}]`) + if count != 1 || !reflect.DeepEqual(boundedEntries(t, entries), want) { + t.Fatalf("depth-capped own members: %d %#v", count, boundedEntries(t, entries)) + } +} + +func TestBuiltInMismatchWalkCaps(t *testing.T) { + t.Run("entries cut at 64 with the uncut count", func(t *testing.T) { + cached, source := map[string]any{}, map[string]any{} + paths := []string{} + for index := 69; index >= 0; index-- { + name := fmt.Sprintf("m%02d", index) + cached[name], source[name] = 0.0, 1.0 + } + for index := 0; index < 64; index++ { + paths = append(paths, fmt.Sprintf("/m%02d", index)) + } + entries, count := walkJSON(cached, source) + if count != 70 || len(entries) != 64 || entries[63].Path != "/m63" { + t.Fatalf("cut: %d entries, count %d", len(entries), count) + } + fields, failure := renderMismatch(false, MismatchDiffPaths, cached, source) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": paths, "mismatchPathCount": 70}, "") + }) + t.Run("pointers clamped to 512 bytes on a UTF-8 boundary", func(t *testing.T) { + names := []string{strings.Repeat("k", 511), strings.Repeat("k", 600), strings.Repeat("é", 300)} + cached, source := map[string]any{}, map[string]any{} + for _, name := range names { + cached[name], source[name] = 1.0, 2.0 + } + fields, failure := renderMismatch(false, MismatchDiffValues, cached, source) + wantPaths := []string{"/" + names[0], "/" + strings.Repeat("k", 497) + ShadowLogTruncationMarker, "/" + strings.Repeat("é", 248) + ShadowLogTruncationMarker} + if failure != "" || !reflect.DeepEqual(fields["mismatchPaths"], wantPaths) || len(wantPaths[1]) != 512 { + t.Fatalf("pointer clamp: %#v", fields["mismatchPaths"]) + } + if entries := entryPaths(parseJSONValue(t, fields["diffJson"].(string)).([]any)); !reflect.DeepEqual(entries, wantPaths) { + t.Fatalf("diffJson pointer clamp: %#v", entries) + } + }) + t.Run("a retained pointer logs as the whole pointer would", func(t *testing.T) { + // The walk copies at most 516 bytes of a kept pointer. Cutting there, + // inside a character or not, must log the same clamped text. + for _, char := range []string{"k", "é", "€", "💡"} { + for pad := 0; pad < len(char); pad++ { + for repeats := 120; repeats <= 540/len(char); repeats++ { + name := strings.Repeat("k", pad) + strings.Repeat(char, repeats) + entries, count := walkJSON(map[string]any{"a": map[string]any{name: 1.0}}, map[string]any{"a": map[string]any{name: 2.0}}) + whole := "/a/" + name + if count != 1 || len(entries[0].Path) > shadowMismatchPointerCopyBytes || clampLogUTF8(entries[0].Path, shadowMismatchPathMaxBytes) != clampLogUTF8(whole, shadowMismatchPathMaxBytes) { + t.Fatalf("a %d-byte pointer kept %q", len(whole), entries[0].Path) + } + } + } + } + }) + t.Run("each side over 1 KiB replaced by its clamped JSON text", func(t *testing.T) { + list := make([]any, 300) + for index := range list { + list[index] = 1000.0 + } + cached := map[string]any{"accent": strings.Repeat("é", 600), "blob": strings.Repeat("x", 2000), "edge": strings.Repeat("e", 1022), "extra": strings.Repeat("z", 1100), "list": list} + source := map[string]any{"accent": "a", "blob": "y", "edge": strings.Repeat("f", 1022), "list": []any{1000.0}} + listJSON, _ := json.Marshal(list) + want := []any{ + map[string]any{"path": "/accent", "cached": `"` + strings.Repeat("é", 504) + ShadowLogTruncationMarker, "source": "a"}, + map[string]any{"path": "/blob", "cached": `"` + strings.Repeat("x", 1009) + ShadowLogTruncationMarker, "source": "y"}, + map[string]any{"path": "/edge", "cached": strings.Repeat("e", 1022), "source": strings.Repeat("f", 1022)}, + map[string]any{"path": "/extra", "cached": `"` + strings.Repeat("z", 1009) + ShadowLogTruncationMarker}, + map[string]any{"path": "/list", "cached": string(listJSON[:1010]) + ShadowLogTruncationMarker, "source": []any{1000.0}}, + } + entries, count := walkJSON(cached, source) + if count != 5 || !reflect.DeepEqual(boundedEntries(t, entries), want) { + t.Fatalf("side clamp: %#v", boundedEntries(t, entries)) + } + }) + t.Run("the walk stops at 32 reference tokens", func(t *testing.T) { + entries, count := walkJSON(nestMembers(40, 1.0), nestMembers(40, 2.0)) + want := []any{map[string]any{"path": strings.Repeat("/n", 32), "cached": nestMembers(8, 1.0), "source": nestMembers(8, 2.0)}} + if count != 1 || !reflect.DeepEqual(boundedEntries(t, entries), want) { + t.Fatalf("depth cap: %#v", boundedEntries(t, entries)) + } + clamped := strings.Repeat(`{"n":`, 202) + ShadowLogTruncationMarker + entries, count = walkJSON(nestMembers(1000, 1.0), nestMembers(1000, 2.0)) + want = []any{map[string]any{"path": strings.Repeat("/n", 32), "cached": clamped, "source": clamped}} + if count != 1 || !reflect.DeepEqual(boundedEntries(t, entries), want) { + t.Fatalf("deep depth cap: %#v", boundedEntries(t, entries)) + } + }) + t.Run("equal containers past the cap compare structurally as deep as JSON parses", func(t *testing.T) { + // encoding/json parses at most 10,000 levels of nesting. + var cached, source any = "leaf", "leaf" + for level := 0; level < 9_999; level++ { + cached = map[string]any{"a": float64(level), "next": cached} + source = map[string]any{"next": source, "a": float64(level)} + } + if entries, count := walkJSON(cached, source); count != 0 || len(entries) != 0 { + t.Fatalf("equal deep containers reported %d entries", count) + } + }) +} + +func TestBuiltInMismatchWalkCopiesBoundedRetainedPointers(t *testing.T) { + // A chain of member names above a wide object and a long array makes every + // pointer below it long. Half their members and elements are equal, and all + // but 64 of the differing ones fall past the cut. Lengthening the names must + // cost only the token stack: never a copy of the prefix per visited member + // or element, which would take over a gigabyte here, nor a whole copy per + // retained entry, about 64 times the prefix, since the warning keeps only + // each pointer's first 512 bytes. + walk := func(nameBytes int) (allocated uint64, prefixBytes int) { + name := strings.Repeat("k", nameBytes) + side := func(differing float64) any { + members, items := map[string]any{}, make([]any, 5_000) + for index := range items { + value := float64(index%2) * differing + members[fmt.Sprintf("m%04d", index)], items[index] = value, value + } + var value any = map[string]any{"items": items, "members": members} + for level := 0; level < 30; level++ { + value = map[string]any{name: value} + } + return value + } + cached, source := side(1), side(2) + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + entries, count := walkJSON(cached, source) + runtime.ReadMemStats(&after) + prefix := strings.Repeat("/"+name, 30) + logged := func(entry MismatchEntry) string { return clampLogUTF8(entry.Path, shadowMismatchPathMaxBytes) } + if count != 5_000 || len(entries) != shadowMismatchMaxEntries || logged(entries[0]) != clampLogUTF8(prefix+"/items/1", shadowMismatchPathMaxBytes) || logged(entries[63]) != clampLogUTF8(prefix+"/items/127", shadowMismatchPathMaxBytes) { + t.Fatalf("walk under %d-byte names: %d entries, count %d", nameBytes, len(entries), count) + } + for _, entry := range entries { + if len(entry.Path) > shadowMismatchPointerCopyBytes { + t.Fatalf("a retained pointer kept %d bytes", len(entry.Path)) + } + } + return after.TotalAlloc - before.TotalAlloc, len(prefix) + } + shortAllocated, shortPrefix := walk(1) + longAllocated, longPrefix := walk(4096) + // The token stack's growth costs about 5 times the prefix it reaches. + if bound := shortAllocated + 16*uint64(longPrefix-shortPrefix); longAllocated > bound { + t.Fatalf("lengthening the pointers by %d bytes allocated %d more bytes, want at most %d", longPrefix-shortPrefix, longAllocated-shortAllocated, bound-shortAllocated) + } +} + +var ( + cachedUser = map[string]any{"id": "123", "email": "old@example.com"} + sourceUser = map[string]any{"id": "123", "email": "new@example.com"} + cachedUserJSON = `{"email":"old@example.com","id":"123"}` + sourceUserJSON = `{"email":"new@example.com","id":"123"}` + failedDiff = map[string]any{"mismatchPaths": nil, "mismatchPathCount": nil, "diffJson": nil} +) + +func withFields(base map[string]any, extra map[string]any) map[string]any { + merged := map[string]any{} + for name, value := range base { + merged[name] = value + } + for name, value := range extra { + merged[name] = value + } + return merged +} + +type failingJSON struct{} + +func (failingJSON) MarshalJSON() ([]byte, error) { return nil, errors.New("not serializable") } + +type panickingJSON struct{} + +func (panickingJSON) MarshalJSON() ([]byte, error) { panic("not serializable") } + +// redacted keeps its text out of JSON, as a type that holds a secret does. +type redacted string + +func (redacted) MarshalJSON() ([]byte, error) { return []byte(`"[redacted]"`), nil } + +// addressRedacted redacts through its pointer only, which encoding/json calls +// when the value is addressable, as a slice element is. +type addressRedacted string + +func (*addressRedacted) MarshalJSON() ([]byte, error) { return []byte(`"[redacted]"`), nil } + +// addressHolder keeps an addressRedacted field, which encoding/json redacts +// whenever the holder itself is addressable. +type addressHolder struct{ Secret addressRedacted } + +// marshalingJSON is a named string whose MarshalJSON returns fixed output, or +// panics when the output is "panic". +type marshalingJSON string + +func (value marshalingJSON) MarshalJSON() ([]byte, error) { + if value == "panic" { + panic("not serializable") + } + return []byte(value), nil +} + +// countingJSON counts its renderings and reports the count in each one. +type countingJSON struct { + email string + renders *int +} + +func (value countingJSON) MarshalJSON() ([]byte, error) { + *value.renders++ + return json.Marshal(map[string]any{"email": value.email, "render": *value.renders}) +} + +func TestShadowMismatchLogFieldsPerLeaves(t *testing.T) { + paths := map[string]any{"mismatchPaths": []string{"/email"}, "mismatchPathCount": 1} + diffJSON := `[{"path":"/email","cached":"old@example.com","source":"new@example.com"}]` + values := map[string]any{"cachedValueJson": cachedUserJSON, "sourceValueJson": sourceUserJSON} + for _, test := range []struct { + name string + value bool + diff MismatchDiffLevel + fields map[string]any + }{ + {"neither leaf", false, MismatchDiffOff, map[string]any{}}, + {"diff paths only", false, MismatchDiffPaths, paths}, + {"diff values without value", false, MismatchDiffValues, withFields(paths, map[string]any{"diffJson": diffJSON})}, + {"value only", true, MismatchDiffOff, values}, + {"value with diff paths", true, MismatchDiffPaths, withFields(paths, values)}, + {"value with diff values", true, MismatchDiffValues, withFields(withFields(paths, values), map[string]any{"diffJson": diffJSON})}, + } { + t.Run(test.name, func(t *testing.T) { + fields, failure := renderMismatch(test.value, test.diff, cachedUser, sourceUser) + expectLog(t, fields, failure, test.fields, "") + }) + } +} + +func TestShadowMismatchLogRendersEachRootOnce(t *testing.T) { + var cachedRenders, sourceRenders int + fields, failure := renderMismatch(true, MismatchDiffValues, countingJSON{"a", &cachedRenders}, countingJSON{"b", &sourceRenders}) + expectLog(t, fields, failure, map[string]any{ + "mismatchPaths": []string{"/email"}, + "mismatchPathCount": 1, + "diffJson": `[{"path":"/email","cached":"a","source":"b"}]`, + "cachedValueJson": `{"email":"a","render":1}`, + "sourceValueJson": `{"email":"b","render":1}`, + }, "") + if cachedRenders != 1 || sourceRenders != 1 { + t.Fatalf("roots rendered %d and %d times", cachedRenders, sourceRenders) + } +} + +func TestShadowMismatchLogNoEntriesForIdenticalRenderings(t *testing.T) { + noEntries := map[string]any{"mismatchPaths": []string{}, "mismatchPathCount": 0, "diffJson": "[]"} + // The default comparator tells an Absent member from a missing one, but + // neither renders. + fields, failure := renderMismatch(true, MismatchDiffValues, map[string]any{"id": 1, "dropped": Absent}, map[string]any{"id": 1}) + expectLog(t, fields, failure, withFields(noEntries, map[string]any{"cachedValueJson": `{"id":1}`, "sourceValueJson": `{"id":1}`}), "") + // A time.Time renders as the ISO string the JSON codec stored. + at := time.Date(2026, 10, 24, 0, 0, 0, 0, time.UTC) + fields, failure = renderMismatch(false, MismatchDiffValues, map[string]any{"at": "2026-10-24T00:00:00Z"}, map[string]any{"at": at}) + expectLog(t, fields, failure, noEntries, "") +} + +func TestShadowMismatchLogAbsentRootIsASide(t *testing.T) { + version := map[string]any{"version": 2} + fields, failure := renderMismatch(true, MismatchDiffValues, Absent, version) + expectLog(t, fields, failure, map[string]any{ + "mismatchPaths": []string{""}, "mismatchPathCount": 1, "diffJson": `[{"path":"","source":{"version":2}}]`, + "cachedValueJson": nil, "sourceValueJson": `{"version":2}`, + }, "") + fields, failure = renderMismatch(true, MismatchDiffValues, version, Absent) + expectLog(t, fields, failure, map[string]any{ + "mismatchPaths": []string{""}, "mismatchPathCount": 1, "diffJson": `[{"path":"","cached":{"version":2}}]`, + "cachedValueJson": `{"version":2}`, "sourceValueJson": nil, + }, "") + fields, failure = renderMismatch(false, MismatchDiffPaths, Absent, nil) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": []string{""}, "mismatchPathCount": 1}, "") + // A stored null is a value, so it stays distinct from absence. + if fields, _ := renderMismatch(false, MismatchDiffValues, Absent, nil); fields["diffJson"] != `[{"path":"","source":null}]` { + t.Fatalf("absent against null: %#v", fields["diffJson"]) + } + // Absence beside a root with no rendering is still a render failure. + fields, failure = renderMismatch(true, MismatchDiffValues, Absent, big.NewInt(1)) + expectLog(t, fields, failure, withFields(failedDiff, map[string]any{"cachedValueJson": nil, "sourceValueJson": nil}), "render_error") + // A custom comparator may call two absent values different; no field differs. + fields, failure = renderMismatch(true, MismatchDiffValues, Absent, Absent) + expectLog(t, fields, failure, map[string]any{ + "mismatchPaths": []string{}, "mismatchPathCount": 0, "diffJson": "[]", "cachedValueJson": nil, "sourceValueJson": nil, + }, "") +} + +func TestShadowMismatchLogRootRenderFailures(t *testing.T) { + cycle := map[string]any{} + cycle["self"] = cycle + for _, test := range []struct { + name string + unrenderable any + }{ + {"cycle", cycle}, {"big integer", big.NewInt(1)}, {"failing MarshalJSON", failingJSON{}}, + {"panicking MarshalJSON", panickingJSON{}}, {"function", func() {}}, {"channel", make(chan int)}, {"complex number", complex(1, 2)}, + } { + t.Run(test.name, func(t *testing.T) { + fields, failure := renderMismatch(true, MismatchDiffValues, test.unrenderable, map[string]any{"version": 2}) + expectLog(t, fields, failure, withFields(failedDiff, map[string]any{"cachedValueJson": nil, "sourceValueJson": `{"version":2}`}), "render_error") + fields, failure = renderMismatch(true, MismatchDiffValues, map[string]any{"version": 2}, test.unrenderable) + expectLog(t, fields, failure, withFields(failedDiff, map[string]any{"cachedValueJson": `{"version":2}`, "sourceValueJson": nil}), "render_error") + fields, failure = renderMismatch(false, MismatchDiffPaths, map[string]any{"version": 2}, test.unrenderable) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": nil, "mismatchPathCount": nil}, "render_error") + }) + } +} + +func TestShadowMismatchLogClampsDiffJSON(t *testing.T) { + cached, source := map[string]any{}, map[string]any{} + paths := []string{} + var entries strings.Builder + entries.WriteString("[") + for index := 0; index < 20; index++ { + name := fmt.Sprintf("m%02d", index) + cached[name], source[name] = strings.Repeat("x", 500), strings.Repeat("y", 500) + paths = append(paths, "/"+name) + if index > 0 { + entries.WriteString(",") + } + fmt.Fprintf(&entries, `{"path":"/%s","cached":"%s","source":"%s"}`, name, strings.Repeat("x", 500), strings.Repeat("y", 500)) + } + // The 8 KiB bound is the record's, so it is written out rather than taken + // from the implementation's constant. + diffJSON := entries.String()[:8192-len(ShadowLogTruncationMarker)] + ShadowLogTruncationMarker + fields, failure := renderMismatch(false, MismatchDiffValues, cached, source) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": paths, "mismatchPathCount": 20, "diffJson": diffJSON}, "") + if len(diffJSON) != 8192 { + t.Fatalf("diffJson bytes: %d", len(diffJSON)) + } + fields, _ = renderMismatch(false, MismatchDiffValues, map[string]any{"blob": strings.Repeat("x", 2000)}, map[string]any{"blob": "y"}) + want := parseJSONValue(t, `[{"path":"/blob","cached":"\"`+strings.Repeat("x", 1009)+ShadowLogTruncationMarker+`","source":"y"}]`) + if !reflect.DeepEqual(parseJSONValue(t, fields["diffJson"].(string)), want) { + t.Fatalf("built-in side clamp: %v", fields["diffJson"]) + } +} + +func TestShadowMismatchLogWalkFailureIsHookError(t *testing.T) { + // A rendering nested past encoding/json's 10,000-level limit does not + // parse back, so the built-in walk fails closed as a hook would. + var deep any = 1.0 + for level := 0; level < 10_001; level++ { + deep = []any{deep} + } + fields, failure := renderMismatch(false, MismatchDiffPaths, deep, 2.0) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": nil, "mismatchPathCount": nil}, "hook_error") + fields, failure = renderMismatch(true, MismatchDiffValues, cachedUser, deep) + if failure != "hook_error" || fields["mismatchPaths"] != nil || fields["diffJson"] != nil || fields["cachedValueJson"] != cachedUserJSON || !strings.HasSuffix(fields["sourceValueJson"].(string), ShadowLogTruncationMarker) { + t.Fatalf("walk failure: %#v %q", fields, failure) + } +} + +func hookReturning(entries []MismatchEntry, err error) (func() ([]MismatchEntry, error), *int) { + calls := 0 + return func() ([]MismatchEntry, error) { calls++; return entries, err }, &calls +} + +func TestShadowMismatchDiffHook(t *testing.T) { + t.Run("replaces the built-in walk", func(t *testing.T) { + hook, calls := hookReturning([]MismatchEntry{ + {Path: "/profile/email", Cached: "old@example.com", Source: "new@example.com"}, + {Path: "/roles", Cached: Absent, Source: Absent}, + }, nil) + fields, failure := renderShadowMismatchLog(true, MismatchDiffValues, map[string]any{"id": 1}, map[string]any{"id": 2}, hook) + expectLog(t, fields, failure, map[string]any{ + "mismatchPaths": []string{"/profile/email", "/roles"}, + "mismatchPathCount": 2, + "diffJson": `[{"path":"/profile/email","cached":"old@example.com","source":"new@example.com"},{"path":"/roles"}]`, + "cachedValueJson": `{"id":1}`, + "sourceValueJson": `{"id":2}`, + }, "") + if *calls != 1 { + t.Fatalf("hook calls: %d", *calls) + } + }) + t.Run("renders neither root when value is off", func(t *testing.T) { + var cachedRenders, sourceRenders int + hook, _ := hookReturning([]MismatchEntry{}, nil) + fields, failure := renderShadowMismatchLog(false, MismatchDiffValues, countingJSON{"a", &cachedRenders}, countingJSON{"b", &sourceRenders}, hook) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": []string{}, "mismatchPathCount": 0, "diffJson": "[]"}, "") + if cachedRenders != 0 || sourceRenders != 0 { + t.Fatal("a hook-supplied diff rendered a root") + } + }) + t.Run("a nil slice is an empty list", func(t *testing.T) { + hook, _ := hookReturning(nil, nil) + fields, failure := renderShadowMismatchLog(false, MismatchDiffValues, cachedUser, sourceUser, hook) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": []string{}, "mismatchPathCount": 0, "diffJson": "[]"}, "") + }) + for _, test := range []struct { + name string + hook func() ([]MismatchEntry, error) + }{ + {"returns an error", func() ([]MismatchEntry, error) { return []MismatchEntry{{Path: "/email"}}, errors.New("hook failed") }}, + {"panics", func() ([]MismatchEntry, error) { panic("hook failed") }}, + } { + t.Run("fails closed with no fallback when the hook "+test.name, func(t *testing.T) { + fields, failure := renderShadowMismatchLog(true, MismatchDiffValues, cachedUser, sourceUser, test.hook) + expectLog(t, fields, failure, withFields(failedDiff, map[string]any{"cachedValueJson": cachedUserJSON, "sourceValueJson": sourceUserJSON}), "hook_error") + fields, failure = renderShadowMismatchLog(false, MismatchDiffPaths, cachedUser, sourceUser, test.hook) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": nil, "mismatchPathCount": nil}, "hook_error") + }) + } + cycle := map[string]any{} + cycle["self"] = cycle + for _, test := range []struct { + name string + unrenderable any + }{ + {"a big integer", big.NewInt(1)}, {"a cycle", cycle}, {"a function", func() {}}, {"a failing MarshalJSON", failingJSON{}}, + // MarshalJSON output is parsed back, never spliced in. + {"a named string whose MarshalJSON panics", marshalingJSON("panic")}, + {"a named string whose MarshalJSON writes no JSON", marshalingJSON("{")}, + {"a number outside double range from MarshalJSON", marshalingJSON("1e400")}, + {"an invalid json.RawMessage", json.RawMessage("{")}, + {"an empty json.RawMessage", json.RawMessage{}}, + {"an invalid json.RawMessage inside a list", []any{1, json.RawMessage("{")}}, + } { + t.Run("keeps the paths and nulls diffJson when a side is "+test.name, func(t *testing.T) { + hook, _ := hookReturning([]MismatchEntry{{Path: "/a", Cached: 1, Source: Absent}, {Path: "/b", Cached: Absent, Source: test.unrenderable}}, nil) + fields, failure := renderShadowMismatchLog(false, MismatchDiffValues, cachedUser, sourceUser, hook) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": []string{"/a", "/b"}, "mismatchPathCount": 2, "diffJson": nil}, "render_error") + fields, failure = renderShadowMismatchLog(false, MismatchDiffPaths, cachedUser, sourceUser, hook) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": []string{"/a", "/b"}, "mismatchPathCount": 2}, "") + }) + } + t.Run("stops rendering sides at the first with no rendering", func(t *testing.T) { + renders := 0 + hook, _ := hookReturning([]MismatchEntry{{Path: "/a", Cached: big.NewInt(1), Source: Absent}, {Path: "/b", Cached: Absent, Source: countingJSON{"later", &renders}}}, nil) + fields, failure := renderShadowMismatchLog(false, MismatchDiffValues, cachedUser, sourceUser, hook) + expectLog(t, fields, failure, map[string]any{"mismatchPaths": []string{"/a", "/b"}, "mismatchPathCount": 2, "diffJson": nil}, "render_error") + if renders != 0 { + t.Fatal("rendered a side after the first failure") + } + }) + t.Run("omits Absent sides and renders the rest as JSON", func(t *testing.T) { + hook, _ := hookReturning([]MismatchEntry{ + {Path: "/a", Cached: Absent, Source: Absent}, + {Path: "/b", Cached: 1, Source: Absent}, + {Path: "/c", Cached: Absent, Source: nil}, + {Path: "/at", Cached: time.Date(2026, 7, 31, 0, 0, 0, 0, time.UTC), Source: map[string]any{"ok": true}}, + }, nil) + fields, _ := renderShadowMismatchLog(false, MismatchDiffValues, cachedUser, sourceUser, hook) + if want := `[{"path":"/a"},{"path":"/b","cached":1},{"path":"/c","source":null},{"path":"/at","cached":"2026-07-31T00:00:00Z","source":{"ok":true}}]`; fields["diffJson"] != want { + t.Fatalf("hook sides: %v", fields["diffJson"]) + } + }) + t.Run("renders sides through MarshalJSON at every depth", func(t *testing.T) { + hook, _ := hookReturning([]MismatchEntry{ + {Path: "/password", Cached: redacted("old secret"), Source: redacted("new secret")}, + {Path: "/passwords", Cached: []redacted{"old secret"}, Source: map[string]redacted{"main": "new secret"}}, + {Path: "/profile", Cached: map[string]any{"password": redacted("old secret"), "history": []any{redacted("older secret")}}, Source: JSONObject{{"password", &[]redacted{"new secret"}[0]}}}, + {Path: "/raw", Cached: json.RawMessage(`{"b":1e2,"a":"<&>"}`), Source: json.RawMessage(nil)}, + // encoding/json calls a pointer's MarshalJSON only on an addressable value. + {Path: "/address", Cached: []addressRedacted{"old secret"}, Source: addressRedacted("new value")}, + {Path: "/holder", Cached: &addressHolder{"old secret"}, Source: []addressHolder{{"new secret"}}}, + {Path: "/held", Cached: map[string]any{"user": &addressHolder{"old secret"}}, Source: addressHolder{"copied value"}}, + }, nil) + fields, failure := renderShadowMismatchLog(false, MismatchDiffValues, cachedUser, sourceUser, hook) + want := `[{"path":"/password","cached":"[redacted]","source":"[redacted]"},` + + `{"path":"/passwords","cached":["[redacted]"],"source":{"main":"[redacted]"}},` + + `{"path":"/profile","cached":{"history":["[redacted]"],"password":"[redacted]"},"source":{"password":"[redacted]"}},` + + `{"path":"/raw","cached":{"a":"<&>","b":100},"source":null},` + + `{"path":"/address","cached":["[redacted]"],"source":"new value"},` + + `{"path":"/holder","cached":{"Secret":"[redacted]"},"source":[{"Secret":"[redacted]"}]},` + + `{"path":"/held","cached":{"user":{"Secret":"[redacted]"}},"source":{"Secret":"copied value"}}]` + if failure != "" || fields["diffJson"] != want { + t.Fatalf("hook sides: %q, %v", fields["diffJson"], failure) + } + }) + t.Run("leaves the roots in JSONCodec's encoding", func(t *testing.T) { + // The value fields and the built-in walk log what the codec stores. + fields, failure := renderShadowMismatchLog(true, MismatchDiffValues, map[string]any{"password": redacted("old")}, json.RawMessage(`{}`), nil) + expectLog(t, fields, failure, map[string]any{ + "mismatchPaths": []string{""}, "mismatchPathCount": 1, + "diffJson": `[{"path":"","cached":{"password":"old"},"source":[123,125]}]`, + "cachedValueJson": `{"password":"old"}`, "sourceValueJson": `[123,125]`, + }, "") + if payload, err := (JSONCodec[any]{}).Encode(redacted("old")); err != nil || string(payload.Bytes) != `"old"` { + t.Fatalf("JSONCodec stored %q, %v", payload.Bytes, err) + } + }) + t.Run("clamps paths and sides and cuts at 64 with the total count", func(t *testing.T) { + clampedPath := "/" + strings.Repeat("k", 497) + ShadowLogTruncationMarker + entries := []MismatchEntry{{Path: "/" + strings.Repeat("k", 600), Cached: strings.Repeat("x", 2000), Source: map[string]any{"ok": true}}} + retained := []string{clampedPath} + wantEntries := []any{map[string]any{"path": clampedPath, "cached": `"` + strings.Repeat("x", 1009) + ShadowLogTruncationMarker, "source": map[string]any{"ok": true}}} + for index := 1; index <= 69; index++ { + path := fmt.Sprintf("/%d", index) + entries = append(entries, MismatchEntry{Path: path, Cached: Absent, Source: Absent}) + if index < 64 { + retained = append(retained, path) + wantEntries = append(wantEntries, map[string]any{"path": path}) + } + } + hook, _ := hookReturning(entries, nil) + fields, failure := renderShadowMismatchLog(false, MismatchDiffValues, cachedUser, sourceUser, hook) + if failure != "" || fields["mismatchPathCount"] != 70 || !reflect.DeepEqual(fields["mismatchPaths"], retained) || !reflect.DeepEqual(parseJSONValue(t, fields["diffJson"].(string)), wantEntries) { + t.Fatalf("hook clamps: %#v", fields) + } + }) + t.Run("is not called when diff is off", func(t *testing.T) { + hook, calls := hookReturning([]MismatchEntry{{Path: "/email"}}, nil) + fields, failure := renderShadowMismatchLog(true, MismatchDiffOff, cachedUser, sourceUser, hook) + expectLog(t, fields, failure, map[string]any{"cachedValueJson": cachedUserJSON, "sourceValueJson": sourceUserJSON}, "") + if *calls != 0 { + t.Fatal("hook ran with diff off") + } + }) + t.Run("keeps the entries when a root has no rendering", func(t *testing.T) { + hook, _ := hookReturning([]MismatchEntry{{Path: "/a", Cached: Absent, Source: Absent}}, nil) + fields, failure := renderShadowMismatchLog(true, MismatchDiffValues, big.NewInt(1), sourceUser, hook) + expectLog(t, fields, failure, map[string]any{ + "mismatchPaths": []string{"/a"}, "mismatchPathCount": 1, "diffJson": `[{"path":"/a"}]`, + "cachedValueJson": nil, "sourceValueJson": sourceUserJSON, + }, "render_error") + }) + t.Run("reports hook_error ahead of render_error", func(t *testing.T) { + hook, _ := hookReturning(nil, errors.New("hook failed")) + fields, failure := renderShadowMismatchLog(true, MismatchDiffValues, cycle, sourceUser, hook) + expectLog(t, fields, failure, withFields(failedDiff, map[string]any{"cachedValueJson": nil, "sourceValueJson": sourceUserJSON}), "hook_error") + }) +} diff --git a/go/shadow_warning_test.go b/go/shadow_warning_test.go new file mode 100644 index 00000000..a54ec443 --- /dev/null +++ b/go/shadow_warning_test.go @@ -0,0 +1,563 @@ +package dialcache + +import ( + "bytes" + "context" + "errors" + "log" + "math/big" + "reflect" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/lan17/DialCache/go/internal/logging" + "github.com/prometheus/client_golang/prometheus" +) + +type mismatchRecord struct { + ID string `json:"id"` + Version int `json:"version"` + Tags []string `json:"tags"` + Extra bool `json:"extra,omitempty"` +} + +var ( + mismatchCachedRecord = mismatchRecord{ID: "123", Version: 1, Tags: []string{"a"}} + mismatchSourceRecord = mismatchRecord{ID: "123", Version: 2, Tags: []string{"a", "b"}, Extra: true} + mismatchPathFields = map[string]any{"mismatchPaths": []string{"/extra", "/tags", "/version"}, "mismatchPathCount": 3} + mismatchDiffFields = withFields(mismatchPathFields, map[string]any{"diffJson": `[{"path":"/extra","source":true},{"path":"/tags","cached":["a"],"source":["a","b"]},{"path":"/version","cached":1,"source":2}]`}) + mismatchValueFields = map[string]any{"cachedValueJson": `{"id":"123","tags":["a"],"version":1}`, "sourceValueJson": `{"extra":true,"id":"123","tags":["a","b"],"version":2}`} + mismatchAllFields = withFields(mismatchDiffFields, mismatchValueFields) +) + +// warningRecorder is the logger and observer of one run: it keeps the +// confirmed-mismatch warnings, the diagnostic events and their interleaving. +type warningRecorder struct { + mu sync.Mutex + warnings []map[string]any + events []Event + sequence []string + panicOnWarn bool +} + +func (r *warningRecorder) Debug(string, any) {} +func (r *warningRecorder) Error(string, any) {} +func (r *warningRecorder) Warn(message string, details any) { + if message != "DialCache shadow validation mismatch" { + return + } + r.mu.Lock() + r.warnings = append(r.warnings, details.(map[string]any)) + r.sequence = append(r.sequence, "warning") + r.mu.Unlock() + if r.panicOnWarn { + panic("logger unavailable") + } +} +func (r *warningRecorder) observe(event Event) { + r.mu.Lock() + defer r.mu.Unlock() + r.events = append(r.events, event) + r.sequence = append(r.sequence, event.Kind) +} +func (r *warningRecorder) outcomes(kind string) []string { + r.mu.Lock() + defer r.mu.Unlock() + outcomes := []string{} + for _, event := range r.events { + if event.Kind == kind { + outcomes = append(outcomes, event.Outcome) + } + } + return outcomes +} +func (r *warningRecorder) configResolutions() []map[string]any { + r.mu.Lock() + defer r.mu.Unlock() + errors := []map[string]any{} + for _, event := range r.events { + if event.Kind == "error" && event.Data["error"] == "config_resolution" { + errors = append(errors, event.Data) + } + } + return errors +} + +// confirmedMismatch serves cached from Redis for key "123" on both shadow +// reads and loads source, so a differing pair confirms a mismatch. +type confirmedMismatch[T any] struct { + useCase string + cached, source T + policy Policy + overlay RuntimePolicy + comparator func(cached, source T) (bool, error) + hook func(cached, source T) ([]MismatchEntry, error) + panicOnWarn bool + viaCached bool + options []Option +} + +// shadowLogging is a remote policy whose shadow cohort admits every key. +func shadowLogging(shadow ShadowPolicy) Policy { + shadow.Ramp = Ptr(100.0) + return Policy{RemoteTTL: time.Minute, Shadow: &shadow} +} + +// run executes one served-hit call inside a synctest bubble and waits for the +// detached shadow job. The frame was written 30.5 seconds earlier. +func (m confirmedMismatch[T]) run(t *testing.T) (*warningRecorder, T) { + t.Helper() + payload, err := (JSONCodec[T]{}).Encode(m.cached) + if err != nil { + t.Fatal(err) + } + frame := Frame{CreatedAtMS: uint64(time.Now().UnixMilli() - 30500), Payload: payload.Bytes} + remote := boundaryRemote{read: func(context.Context) (ReadResult, error) { return ReadResult{Kind: "hit", Frame: frame}, nil }} + recorder := &warningRecorder{panicOnWarn: m.panicOnWarn} + options := append([]Option{WithRemote(remote), WithLogger(recorder), WithObserver(recorder.observe), WithShadowOutcomes(func(Event) {})}, m.options...) + if m.overlay != nil { + options = append(options, WithPolicyProvider(func(context.Context, Identity) (RuntimePolicy, error) { return m.overlay, nil })) + } + cache := MustNew(options...) + op := Operation[T]{Identity: Identity{KeyType: "user_id", ID: "123", UseCase: m.useCase}, Policy: m.policy, Comparator: m.comparator, MismatchDiff: m.hook} + load := func(context.Context) (T, error) { return m.source, nil } + var result T + if err := cache.WithEnabled(context.Background(), func(ctx context.Context) error { + if !m.viaCached { + result, err = GetOrLoad(ctx, cache, op, load) + return err + } + read, err := Cached(cache, op, func(string) (Identity, error) { return Identity{ID: "123"}, nil }, func(ctx context.Context, _ string) (T, error) { return load(ctx) }) + if err != nil { + return err + } + result, err = read(ctx, "123") + return err + }); err != nil { + t.Fatal(err) + } + synctest.Wait() + return recorder, result +} + +// mismatchWarning is a warning's metadata for key "123" with the frame's age, +// plus the fields the leaves select. +func mismatchWarning(useCase string, fields map[string]any) map[string]any { + return withFields(map[string]any{ + "cacheNamespace": "urn", "useCase": useCase, "keyType": "user_id", "outcome": "mismatch", + "cachedValueAgeSeconds": 30.5, "cacheKey": "urn:user_id:123#" + useCase, + }, fields) +} + +// expectWarning checks the run's one warning and its one logging outcome. +func expectWarning(t *testing.T, recorder *warningRecorder, warning map[string]any, outcome string) { + t.Helper() + if verdicts := recorder.outcomes("shadowValidation"); !reflect.DeepEqual(verdicts, []string{"mismatch"}) { + t.Fatalf("verdicts: %v", verdicts) + } + if len(recorder.warnings) != 1 || !reflect.DeepEqual(recorder.warnings[0], warning) { + t.Fatalf("warnings: %#v\nwant %#v", recorder.warnings, warning) + } + if outcomes := recorder.outcomes("shadowMismatchLogging"); !reflect.DeepEqual(outcomes, []string{outcome}) { + t.Fatalf("logging outcomes: %v, want %s", outcomes, outcome) + } +} + +func expectNoWarning(t *testing.T, recorder *warningRecorder) { + t.Helper() + if len(recorder.warnings) != 0 || len(recorder.outcomes("shadowMismatchLogging")) != 0 { + t.Fatalf("logging off emitted %v and %v", recorder.warnings, recorder.outcomes("shadowMismatchLogging")) + } +} + +func TestShadowMismatchWarningIsOptIn(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + recorder, result := confirmedMismatch[mismatchRecord]{useCase: "LoggingOff", cached: mismatchCachedRecord, source: mismatchSourceRecord, policy: shadowLogging(ShadowPolicy{})}.run(t) + if !reflect.DeepEqual(result, mismatchCachedRecord) || !reflect.DeepEqual(recorder.outcomes("shadowValidation"), []string{"mismatch"}) { + t.Fatalf("served %v with verdicts %v", result, recorder.outcomes("shadowValidation")) + } + expectNoWarning(t, recorder) + // A match with logging on emits nothing either. + recorder, _ = confirmedMismatch[mismatchRecord]{useCase: "LoggingMatch", cached: mismatchCachedRecord, source: mismatchCachedRecord, policy: shadowLogging(ShadowPolicy{LogMismatches: Ptr(true)})}.run(t) + if !reflect.DeepEqual(recorder.outcomes("shadowValidation"), []string{"match"}) { + t.Fatalf("verdicts: %v", recorder.outcomes("shadowValidation")) + } + expectNoWarning(t, recorder) + }) +} + +func TestShadowMismatchWarningFieldsPerLeaves(t *testing.T) { + hookCalls := 0 + countingHook := func(mismatchRecord, mismatchRecord) ([]MismatchEntry, error) { + hookCalls++ + return []MismatchEntry{{Path: "/hook", Cached: Absent, Source: Absent}}, nil + } + pathsLevel, valuesLevel := Ptr(MismatchDiffPaths), Ptr(MismatchDiffValues) + for _, test := range []struct { + name string + static ShadowPolicy + overlay RuntimePolicy + fields map[string]any + }{ + {"logMismatches true", ShadowPolicy{LogMismatches: Ptr(true)}, nil, mismatchAllFields}, + {"diff paths", ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: pathsLevel}}, nil, mismatchPathFields}, + {"diff values with value false", ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: valuesLevel, Value: Ptr(false)}}, nil, mismatchDiffFields}, + {"value alone", ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Value: Ptr(true)}}, nil, mismatchValueFields}, + // The record's four rows. + {"logMismatches true beside value false", ShadowPolicy{LogMismatches: Ptr(true), MismatchLogging: &MismatchLoggingPolicy{Value: Ptr(false)}}, nil, mismatchDiffFields}, + {"logMismatches false beside diff paths", ShadowPolicy{LogMismatches: Ptr(false), MismatchLogging: &MismatchLoggingPolicy{Diff: pathsLevel}}, nil, mismatchPathFields}, + {"a runtime logMismatches false over a static diff paths", ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: pathsLevel}}, &PolicyOverlay{Shadow: &ShadowPolicy{LogMismatches: Ptr(false)}}, nil}, + {"a runtime logMismatches true over a static value false", ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Value: Ptr(false)}}, JSONPolicy{"shadow": map[string]any{"logMismatches": true}}, mismatchAllFields}, + } { + t.Run(test.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + run := confirmedMismatch[mismatchRecord]{useCase: "Leaves", cached: mismatchCachedRecord, source: mismatchSourceRecord, policy: shadowLogging(test.static), overlay: test.overlay} + if test.name == "value alone" { + run.hook = countingHook + } + recorder, _ := run.run(t) + if len(recorder.configResolutions()) != 0 { + t.Fatalf("valid leaves recorded %v", recorder.configResolutions()) + } + if test.fields == nil { + expectNoWarning(t, recorder) + return + } + expectWarning(t, recorder, mismatchWarning("Leaves", test.fields), "logged") + // The warning follows the verdict and its age; its one outcome follows it. + if want := []string{"shadowValidation", "shadowAge", "warning", "shadowMismatchLogging"}; !reflect.DeepEqual(recorder.sequence[len(recorder.sequence)-4:], want) { + t.Fatalf("sequence: %v", recorder.sequence) + } + event := recorder.events[len(recorder.events)-1] + if want := map[string]any{"cacheNamespace": "urn", "useCase": "Leaves", "keyType": "user_id", "outcome": "logged"}; !reflect.DeepEqual(event.Data, want) || event.Key != "urn:user_id:123#Leaves" { + t.Fatalf("logging event: %+v", event) + } + }) + }) + } + // A subtest, so the mutation runner credits this assertion to a test. + t.Run("diff off never runs the hook", func(t *testing.T) { + if hookCalls != 0 { + t.Fatalf("diff off still ran the hook %d times", hookCalls) + } + }) +} + +func TestShadowMismatchWarningOnRampedDownPath(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + policy := shadowLogging(ShadowPolicy{LogMismatches: Ptr(true)}) + policy.RemoteRamp = Ptr(0.0) + recorder, result := confirmedMismatch[mismatchRecord]{useCase: "RampedDown", cached: mismatchCachedRecord, source: mismatchSourceRecord, policy: policy}.run(t) + if !reflect.DeepEqual(result, mismatchSourceRecord) { + t.Fatalf("ramped-down caller got %v", result) + } + expectWarning(t, recorder, mismatchWarning("RampedDown", mismatchAllFields), "logged") + }) +} + +func TestShadowMismatchDiffHookThroughCachedAndGetOrLoad(t *testing.T) { + for _, viaCached := range []bool{false, true} { + t.Run(map[bool]string{false: "GetOrLoad", true: "Cached"}[viaCached], func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var calls [][2]mismatchRecord + hook := func(cached, source mismatchRecord) ([]MismatchEntry, error) { + calls = append(calls, [2]mismatchRecord{cached, source}) + return []MismatchEntry{ + {Path: "/custom", Cached: "c", Source: "s"}, + {Path: "/only-path", Cached: Absent, Source: Absent}, + {Path: "/one-side", Cached: Absent, Source: map[string]any{"nested": true}}, + }, nil + } + recorder, _ := confirmedMismatch[mismatchRecord]{useCase: "Hook", cached: mismatchCachedRecord, source: mismatchSourceRecord, policy: shadowLogging(ShadowPolicy{LogMismatches: Ptr(true)}), hook: hook, viaCached: viaCached}.run(t) + if !reflect.DeepEqual(calls, [][2]mismatchRecord{{mismatchCachedRecord, mismatchSourceRecord}}) { + t.Fatalf("hook calls: %v", calls) + } + expectWarning(t, recorder, mismatchWarning("Hook", withFields(mismatchValueFields, map[string]any{ + "mismatchPaths": []string{"/custom", "/only-path", "/one-side"}, + "mismatchPathCount": 3, + "diffJson": `[{"path":"/custom","cached":"c","source":"s"},{"path":"/only-path"},{"path":"/one-side","source":{"nested":true}}]`, + })), "logged") + }) + }) + } +} + +func TestShadowMismatchWarningFailureOutcomes(t *testing.T) { + hookError := func(mismatchRecord, mismatchRecord) ([]MismatchEntry, error) { + return nil, errors.New("diff hook unavailable") + } + hookPanic := func(mismatchRecord, mismatchRecord) ([]MismatchEntry, error) { panic("diff hook unavailable") } + unrenderableSide := func(mismatchRecord, mismatchRecord) ([]MismatchEntry, error) { + return []MismatchEntry{{Path: "/version", Cached: big.NewInt(1), Source: Absent}}, nil + } + panickingSide := func(mismatchRecord, mismatchRecord) ([]MismatchEntry, error) { + return []MismatchEntry{{Path: "/version", Cached: Absent, Source: marshalingJSON("panic")}}, nil + } + failedDiff := map[string]any{"mismatchPaths": nil, "mismatchPathCount": nil, "diffJson": nil} + for _, test := range []struct { + name string + leaves ShadowPolicy + hook func(mismatchRecord, mismatchRecord) ([]MismatchEntry, error) + panicOnWarn bool + fields map[string]any + outcome string + }{ + {"the hook returns an error", ShadowPolicy{LogMismatches: Ptr(true)}, hookError, false, withFields(failedDiff, mismatchValueFields), "hook_error"}, + {"the hook panics", ShadowPolicy{LogMismatches: Ptr(true)}, hookPanic, false, withFields(failedDiff, mismatchValueFields), "hook_error"}, + {"the logger panics", ShadowPolicy{LogMismatches: Ptr(true)}, nil, true, mismatchAllFields, "logger_error"}, + {"the hook fails and the logger panics", ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: Ptr(MismatchDiffValues)}}, hookError, true, failedDiff, "hook_error"}, + {"a hook side has no rendering and the logger panics", ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: Ptr(MismatchDiffValues)}}, unrenderableSide, true, map[string]any{"mismatchPaths": []string{"/version"}, "mismatchPathCount": 1, "diffJson": nil}, "render_error"}, + {"a hook side's MarshalJSON panics", ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: Ptr(MismatchDiffValues)}}, panickingSide, false, map[string]any{"mismatchPaths": []string{"/version"}, "mismatchPathCount": 1, "diffJson": nil}, "render_error"}, + } { + t.Run(test.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + recorder, result := confirmedMismatch[mismatchRecord]{useCase: "Failure", cached: mismatchCachedRecord, source: mismatchSourceRecord, policy: shadowLogging(test.leaves), hook: test.hook, panicOnWarn: test.panicOnWarn}.run(t) + if !reflect.DeepEqual(result, mismatchCachedRecord) { + t.Fatalf("warning failure changed the result: %v", result) + } + fields := test.fields + if test.outcome != "logger_error" { + // The logger never received a logger_error, so it carries none. + fields = withFields(fields, map[string]any{"loggingOutcome": test.outcome}) + } + expectWarning(t, recorder, mismatchWarning("Failure", fields), test.outcome) + }) + }) + } +} + +func TestShadowMismatchWarningRenderFailure(t *testing.T) { + // A source with no JSON rendering nulls its value field and the built-in + // diff fields, under the default comparator and a custom one. + for _, comparator := range []func(any, any) (bool, error){nil, func(any, any) (bool, error) { return false, nil }} { + synctest.Test(t, func(t *testing.T) { + recorder, _ := confirmedMismatch[any]{useCase: "RenderFailure", cached: map[string]any{"id": "123", "version": 1}, source: failingJSON{}, policy: shadowLogging(ShadowPolicy{LogMismatches: Ptr(true)}), comparator: comparator}.run(t) + expectWarning(t, recorder, mismatchWarning("RenderFailure", map[string]any{ + "mismatchPaths": nil, "mismatchPathCount": nil, "diffJson": nil, + "cachedValueJson": `{"id":"123","version":1}`, "sourceValueJson": nil, "loggingOutcome": "render_error", + }), "render_error") + }) + } + // A hook at diff paths renders neither value, so the same source logs. + synctest.Test(t, func(t *testing.T) { + hook := func(any, any) ([]MismatchEntry, error) { + return []MismatchEntry{{Path: "/version", Cached: Absent, Source: Absent}}, nil + } + recorder, _ := confirmedMismatch[any]{useCase: "HookPaths", cached: map[string]any{"id": "123"}, source: failingJSON{}, policy: shadowLogging(ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: Ptr(MismatchDiffPaths)}}), hook: hook}.run(t) + expectWarning(t, recorder, mismatchWarning("HookPaths", map[string]any{"mismatchPaths": []string{"/version"}, "mismatchPathCount": 1}), "logged") + }) +} + +func TestShadowMismatchWarningDefaultLogger(t *testing.T) { + // Pointers carry member names as written. The default logger writes the + // details as JSON, so each pointer keeps its bounds, and escapes the whole + // line, so a newline or terminal escape in a name cannot forge a line. + for _, test := range []struct { + name string + cached, source any + paths string + }{ + {"keeps a forged line on one line", map[string]any{"a\nDialCache forged\x1b[2J": 1}, map[string]any{"a\nDialCache forged\x1b[2J": 2}, `["/a\nDialCache forged\u001b[2J"]`}, + {"quotes the root pointer", 1, "1", `[""]`}, + {"keeps a pointer holding a space one element", map[string]any{"a b": 1}, map[string]any{"a b": 2}, `["/a b"]`}, + } { + t.Run(test.name, func(t *testing.T) { + var output bytes.Buffer + writer, flags := log.Writer(), log.Flags() + log.SetOutput(&output) + log.SetFlags(0) + defer func() { log.SetOutput(writer); log.SetFlags(flags) }() + synctest.Test(t, func(t *testing.T) { + recorder, _ := confirmedMismatch[any]{useCase: "DefaultLogger", cached: test.cached, source: test.source, + policy: shadowLogging(ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: Ptr(MismatchDiffPaths)}}), options: []Option{WithLogger(logging.Logger{})}}.run(t) + if outcomes := recorder.outcomes("shadowMismatchLogging"); !reflect.DeepEqual(outcomes, []string{"logged"}) { + t.Fatalf("logging outcomes: %v", outcomes) + } + }) + want := `DialCache shadow validation mismatch: {"cacheKey":"urn:user_id:123#DefaultLogger","cacheNamespace":"urn","cachedValueAgeSeconds":30.5,"keyType":"user_id","mismatchPathCount":1,"mismatchPaths":` + test.paths + `,"outcome":"mismatch","useCase":"DefaultLogger"}` + "\n" + if output.String() != want { + t.Fatalf("logged %q, want %q", output.String(), want) + } + }) + } +} + +func TestShadowMismatchWarningAbsentCachedValue(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // The default codec stores a negative-cache entry as Absent. + recorder, result := confirmedMismatch[any]{useCase: "AbsentCached", cached: Absent, source: map[string]any{"id": "123"}, policy: shadowLogging(ShadowPolicy{LogMismatches: Ptr(true)})}.run(t) + if !IsAbsent(result) { + t.Fatalf("served %v", result) + } + expectWarning(t, recorder, mismatchWarning("AbsentCached", map[string]any{ + "mismatchPaths": []string{""}, "mismatchPathCount": 1, "diffJson": `[{"path":"","source":{"id":"123"}}]`, + "cachedValueJson": nil, "sourceValueJson": `{"id":"123"}`, + }), "logged") + }) +} + +func TestShadowMismatchWarningRuntimeConfig(t *testing.T) { + configResolution := map[string]any{"cacheNamespace": "urn", "useCase": "Config", "keyType": "user_id", "layer": "remote", "error": "config_resolution", "inFallback": false} + for _, test := range []struct { + name string + static ShadowPolicy + overlay RuntimePolicy + errors int + fields map[string]any + }{ + {"an invalid runtime shorthand fails closed", ShadowPolicy{LogMismatches: Ptr(true)}, JSONPolicy{"shadow": map[string]any{"logMismatches": "yes"}}, 1, nil}, + {"an invalid runtime leaf keeps the valid static leaf", ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Value: Ptr(true)}}, JSONPolicy{"shadow": map[string]any{"mismatchLogging": map[string]any{"diff": "everything"}}}, 1, mismatchValueFields}, + {"a malformed runtime shorthand logs the leaves beside it", ShadowPolicy{LogMismatches: Ptr(true)}, JSONPolicy{"shadow": map[string]any{"logMismatches": "yes", "mismatchLogging": map[string]any{"value": false, "diff": "paths"}}}, 1, mismatchPathFields}, + {"a typed invalid diff level acts off", ShadowPolicy{LogMismatches: Ptr(true)}, &PolicyOverlay{Shadow: &ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: Ptr(MismatchDiffLevel("all"))}}}, 1, mismatchValueFields}, + {"a runtime logMismatches false kills every inherited leaf", ShadowPolicy{LogMismatches: Ptr(true), MismatchLogging: &MismatchLoggingPolicy{Value: Ptr(true), Diff: Ptr(MismatchDiffValues)}}, &PolicyOverlay{Shadow: &ShadowPolicy{LogMismatches: Ptr(false)}}, 0, nil}, + } { + t.Run(test.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + recorder, _ := confirmedMismatch[mismatchRecord]{useCase: "Config", cached: mismatchCachedRecord, source: mismatchSourceRecord, policy: shadowLogging(test.static), overlay: test.overlay}.run(t) + if errors := recorder.configResolutions(); len(errors) != test.errors || (test.errors == 1 && !reflect.DeepEqual(errors[0], configResolution)) { + t.Fatalf("config_resolution errors: %v", errors) + } + if test.fields == nil { + if !reflect.DeepEqual(recorder.outcomes("shadowValidation"), []string{"mismatch"}) { + t.Fatalf("verdicts: %v", recorder.outcomes("shadowValidation")) + } + expectNoWarning(t, recorder) + return + } + expectWarning(t, recorder, mismatchWarning("Config", test.fields), "logged") + }) + }) + } +} + +func TestShadowMismatchLoggingConfigErrorOncePerAdmittedJob(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + payload, _ := (JSONCodec[mismatchRecord]{}).Encode(mismatchCachedRecord) + frame := Frame{CreatedAtMS: uint64(time.Now().UnixMilli()), Payload: payload.Bytes} + remote := boundaryRemote{read: func(context.Context) (ReadResult, error) { return ReadResult{Kind: "hit", Frame: frame}, nil }} + recorder := &warningRecorder{} + overlay := JSONPolicy{"shadow": map[string]any{"logMismatches": "yes", "mismatchLogging": map[string]any{"value": "no", "diff": "all"}}} + cache := MustNew(WithRemote(remote), WithLogger(recorder), WithObserver(recorder.observe), WithShadowOutcomes(func(Event) {}), + WithPolicyProvider(func(context.Context, Identity) (RuntimePolicy, error) { return overlay, nil })) + op := Operation[mismatchRecord]{Identity: Identity{KeyType: "user_id", ID: "123", UseCase: "Config"}, Policy: shadowLogging(ShadowPolicy{})} + release := make(chan struct{}) + load := func(context.Context) (mismatchRecord, error) { <-release; return mismatchSourceRecord, nil } + call := func() { + if err := cache.WithEnabled(context.Background(), func(ctx context.Context) error { + _, err := GetOrLoad(ctx, cache, op, load) + return err + }); err != nil { + t.Error(err) + } + } + // The first served hit admits a job whose source blocks; the second + // finds that job holding the key and is dropped before admission. + call() + synctest.Wait() + call() + close(release) + synctest.Wait() + if verdicts := recorder.outcomes("shadowValidation"); !reflect.DeepEqual(verdicts, []string{"dropped", "mismatch"}) { + t.Fatalf("verdicts: %v", verdicts) + } + // Three invalid inputs, one admitted job: one config_resolution. + if errors := recorder.configResolutions(); len(errors) != 1 { + t.Fatalf("config_resolution errors: %v", errors) + } + expectNoWarning(t, recorder) + }) +} + +func TestShadowMismatchWarningKeepsTheShadowSlot(t *testing.T) { + // A confirmed mismatch keeps its slot until its warning is logged, so + // capacity bounds a slow hook as it bounds a slow comparator. + synctest.Test(t, func(t *testing.T) { + payload, _ := (JSONCodec[mismatchRecord]{}).Encode(mismatchCachedRecord) + frame := Frame{CreatedAtMS: uint64(time.Now().UnixMilli()), Payload: payload.Bytes} + remote := boundaryRemote{read: func(context.Context) (ReadResult, error) { return ReadResult{Kind: "hit", Frame: frame}, nil }} + recorder := &warningRecorder{} + cache := MustNew(WithRemote(remote), WithLogger(recorder), WithObserver(recorder.observe), WithShadowOutcomes(func(Event) {}), WithShadowCapacity(1)) + release := make(chan struct{}) + hook := func(mismatchRecord, mismatchRecord) ([]MismatchEntry, error) { <-release; return nil, nil } + call := func(id string) { + op := Operation[mismatchRecord]{Identity: Identity{KeyType: "user_id", ID: id, UseCase: "Capacity"}, MismatchDiff: hook, + Policy: shadowLogging(ShadowPolicy{MismatchLogging: &MismatchLoggingPolicy{Diff: Ptr(MismatchDiffPaths)}})} + if err := cache.WithEnabled(context.Background(), func(ctx context.Context) error { + _, err := GetOrLoad(ctx, cache, op, func(context.Context) (mismatchRecord, error) { return mismatchSourceRecord, nil }) + return err + }); err != nil { + t.Error(err) + } + } + // The first job confirms its mismatch and blocks in the hook; a job for + // another key finds no free slot. + call("123") + synctest.Wait() + call("456") + synctest.Wait() + verdicts := recorder.outcomes("shadowValidation") + close(release) + synctest.Wait() + if !reflect.DeepEqual(verdicts, []string{"mismatch", "dropped"}) { + t.Fatalf("verdicts while the hook runs: %v", verdicts) + } + // Once the warning is logged, the slot admits the next job. + call("789") + synctest.Wait() + if verdicts := recorder.outcomes("shadowValidation"); !reflect.DeepEqual(verdicts, []string{"mismatch", "dropped", "mismatch"}) { + t.Fatalf("verdicts: %v", verdicts) + } + if outcomes := recorder.outcomes("shadowMismatchLogging"); !reflect.DeepEqual(outcomes, []string{"logged", "logged"}) { + t.Fatalf("logging outcomes: %v", outcomes) + } + }) +} + +func TestShadowMismatchLoggingExporters(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + registry := prometheus.NewRegistry() + exporter, err := NewPrometheusMetrics(registry, "") + if err != nil { + t.Fatal(err) + } + client := &recordingDogStatsD{} + datadog, err := NewDatadogMetrics(DatadogMetricsOptions{Client: client, ObservationMetricType: "distribution"}) + if err != nil { + t.Fatal(err) + } + run := confirmedMismatch[mismatchRecord]{useCase: "Exported", cached: mismatchCachedRecord, source: mismatchSourceRecord, policy: shadowLogging(ShadowPolicy{LogMismatches: Ptr(true)}), panicOnWarn: true, options: []Option{WithMetrics(exporter), WithMetrics(datadog)}} + if recorder, _ := run.run(t); !reflect.DeepEqual(recorder.outcomes("shadowMismatchLogging"), []string{"logger_error"}) { + t.Fatalf("logging outcomes: %v", recorder.outcomes("shadowMismatchLogging")) + } + labels := map[string]string{"cache_namespace": "urn", "use_case": "Exported", "key_type": "user_id", "outcome": "logger_error"} + families, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + exported := false + for _, family := range families { + if family.GetName() != "dialcache_shadow_mismatch_logging_counter" { + continue + } + got := map[string]string{} + for _, label := range family.Metric[0].Label { + got[label.GetName()] = label.GetValue() + } + exported = len(family.Metric) == 1 && family.Metric[0].Counter.GetValue() == 1 && reflect.DeepEqual(got, labels) + } + if !exported { + t.Fatal("Prometheus did not count the logging outcome") + } + counted := false + for _, metric := range client.events { + if metric.name == "dialcache.shadow_mismatch_logging.count" { + counted = metric.method == "increment" && metric.value == 1 && reflect.DeepEqual(metric.tags, labels) + } + } + if !counted { + t.Fatal("Datadog did not count the logging outcome") + } + }) +} diff --git a/python/dialcache/_callbacks.py b/python/dialcache/_callbacks.py new file mode 100644 index 00000000..2c65bf51 --- /dev/null +++ b/python/dialcache/_callbacks.py @@ -0,0 +1,27 @@ +"""Isolation for application callbacks whose contract is synchronous.""" + +from __future__ import annotations + +import asyncio +import contextvars +import inspect +from typing import Any + + +def discard_awaitable(value: Any) -> None: + """Fail closed on an awaitable a synchronous callback returned. + + A rejected result must not start work or take ownership of an + application's existing task: a coroutine that has not started is closed, + and a future's exception is consumed without awaiting it. + """ + if inspect.iscoroutine(value): + if inspect.getcoroutinestate(value) == inspect.CORO_CREATED: + value.close() + elif asyncio.isfuture(value): + + def consume(done: asyncio.Future[Any]) -> None: + if not done.cancelled(): + done.exception() + + value.add_done_callback(consume, context=contextvars.Context()) diff --git a/python/dialcache/_shadow_log.py b/python/dialcache/_shadow_log.py new file mode 100644 index 00000000..277e0500 --- /dev/null +++ b/python/dialcache/_shadow_log.py @@ -0,0 +1,324 @@ +"""Bounded content of confirmed shadow-mismatch warnings. + +Every field derives from one native JSON rendering of each compared value, so +a failure logs less, never more. The built-in difference computation walks the +two parsed renderings and reports RFC 6901 JSON Pointers. +""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import math +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from ._callbacks import discard_awaitable +from .protocol import utf8_bytes +from .serializer import UNDEFINED, _native_json + +KEY_MAX_BYTES = 2 * 1024 +VALUE_MAX_BYTES = 8 * 1024 +DIFF_MAX_BYTES = 8 * 1024 +MAX_ENTRIES = 64 +PATH_MAX_BYTES = 512 +SIDE_MAX_BYTES = 1024 +# Reference tokens in the deepest pointer the built-in walk reports. +MAX_DEPTH = 32 +TRUNCATION_MARKER = "...[truncated]" +_MARKER_BYTES = TRUNCATION_MARKER.encode() + +# One difference: a pointer, and each side under "cached" or "source" when the +# value has that side. A missing key is an absent side; None is JSON null. +Entry = dict[str, Any] + + +@dataclass(frozen=True) +class MismatchLogPlan: + """The effective content leaves captured when a shadow job is admitted.""" + + value: bool + diff: str + + +def clamp_utf8(text: str, limit: int) -> str: + """Clamp to ``limit`` UTF-8 bytes with the marker, never splitting a code point.""" + encoded = utf8_bytes(text) + if len(encoded) <= limit: + return text + return encoded[: limit - len(_MARKER_BYTES)].decode("utf-8", "ignore") + TRUNCATION_MARKER + + +def render_json(value: Any) -> str | None: + """The native JSON rendering ``JsonSerializer`` stores, or None for a value without one. + + Lone surrogates are escaped, so the rendering is valid UTF-8. A top-level + ``UNDEFINED`` has none here: its stored sentinel is not JSON. + """ + try: + return _native_json(value) + except Exception: + return None + + +def render_shadow_mismatch_log( + plan: MismatchLogPlan, + cached: Any, + source: Any, + hook: Callable[[Any, Any], Any] | None = None, +) -> tuple[dict[str, Any], str | None]: + """Build the content fields the plan requests for one confirmed mismatch. + + Returns the fields, each requested one present and None when it failed + closed, and the failure: ``"hook_error"`` for the hook, which ranks above + ``"render_error"`` for JSON rendering, or None. Each side is rendered at + most once. A failed hook never falls back to the built-in walk. A + top-level ``UNDEFINED``, which a negative-cache entry stores, is an absent + side rather than a failure: its value field is None, and the walk reports + the other side whole at ``""``. + """ + render_sides = plan.value or (plan.diff != "off" and hook is None) + cached_json = render_json(cached) if render_sides else None + source_json = render_json(source) if render_sides else None + diff_failed = False + render_failed = render_sides and ( + (cached_json is None and cached is not UNDEFINED) or (source_json is None and source is not UNDEFINED) + ) + + fields: dict[str, Any] = {} + if plan.diff != "off": + with_values = plan.diff == "values" + diff: tuple[list[Entry], int] | None = None + if hook is not None: + diff = _run_hook(hook, cached, source) + diff_failed = diff is None + elif not render_failed: + diff = _walk_rendered(cached_json, source_json) + if diff is None: + fields["mismatchPaths"] = None + fields["mismatchPathCount"] = None + if with_values: + fields["diffJson"] = None + else: + entries, count = diff + diff_json = None + if with_values: + diff_json = _render_diff_json(entries) + render_failed = render_failed or diff_json is None + # The paths level never renders a side. + fields["mismatchPaths"] = [clamp_utf8(entry["path"], PATH_MAX_BYTES) for entry in entries] + fields["mismatchPathCount"] = count + if with_values: + fields["diffJson"] = diff_json + + if plan.value: + fields["cachedValueJson"] = None if cached_json is None else clamp_utf8(cached_json, VALUE_MAX_BYTES) + fields["sourceValueJson"] = None if source_json is None else clamp_utf8(source_json, VALUE_MAX_BYTES) + return fields, "hook_error" if diff_failed else "render_error" if render_failed else None + + +def diff_shadow_mismatch_json(cached: Any, source: Any) -> tuple[list[Entry | None], int]: + """The built-in difference computation over two JSON values. + + Objects recurse over the union of their member names in UTF-16 code-unit + order, and a member on one side only is reported with that side. Arrays of + equal length recurse by index; arrays of different length are reported once + at their own pointer. Any other differing pair, roots of different kinds + included, is reported with both sides, and a differing pair of containers + at ``MAX_DEPTH`` reference tokens is reported whole. Entries keep walk + order and are cut at ``MAX_ENTRIES``; the count is the uncut total. Each + pointer and side is clamped as in ``diffJson``. + """ + entries, count = _walk(cached, source) + # JSON values always render, so every entry bounds. + return [_bound_entry(entry) for entry in entries], count + + +def _walk_rendered(cached_json: str | None, source_json: str | None) -> tuple[list[Entry], int]: + # None is an absent side; a side that failed to render never reaches here. + # Each string came from a successful native rendering, so this is the one + # boundary from arbitrary values to the closed JSON domain. + if cached_json is None: + return ([], 0) if source_json is None else ([{"path": "", "source": json.loads(source_json)}], 1) + if source_json is None: + return [{"path": "", "cached": json.loads(cached_json)}], 1 + if cached_json == source_json: + return [], 0 + return _walk(json.loads(cached_json), json.loads(source_json)) + + +def _walk(cached: Any, source: Any) -> tuple[list[Entry], int]: + entries: list[Entry] = [] + count = 0 + # The escaped reference tokens down to the pair being walked. Python + # strings have no ropes, so a pointer is built only for a retained entry: + # copying the prefix for each member would cost members times its length. + tokens: list[str] = [] + + def report(**sides: Any) -> None: + nonlocal count + count += 1 + if len(entries) < MAX_ENTRIES: + entries.append({"path": _clamped_pointer(tokens), **sides}) + + def walk(left: Any, right: Any) -> None: + kind = _kind(left) + if kind != _kind(right) or kind == "primitive": + if not _same_primitive(left, right): + report(cached=left, source=right) + return + if len(tokens) >= MAX_DEPTH: + if not _json_equal(left, right): + report(cached=left, source=right) + return + if kind == "array": + if len(left) != len(right): + report(cached=left, source=right) + return + for index, (item, other) in enumerate(zip(left, right)): + tokens.append(str(index)) + walk(item, other) + tokens.pop() + return + for name in sorted(left.keys() | right.keys(), key=_utf16_order): + tokens.append(_escape_token(name)) + if name not in right: + report(cached=left[name]) + elif name not in left: + report(source=right[name]) + else: + walk(left[name], right[name]) + tokens.pop() + + walk(cached, source) + return entries, count + + +def _clamped_pointer(tokens: list[str]) -> str: + # A token longer than the cap in code points is longer than it in UTF-8 + # bytes, so cutting each token there keeps every byte the clamp keeps. + return clamp_utf8("".join(f"/{token[:PATH_MAX_BYTES]}" for token in tokens), PATH_MAX_BYTES) + + +def _json_equal(cached: Any, source: Any) -> bool: + # Structural equality for a depth-capped pair, iterative so that a subtree + # as deep as native JSON rendering allows cannot exhaust the stack. + pending = [(cached, source)] + while pending: + left, right = pending.pop() + kind = _kind(left) + if kind != _kind(right): + return False + if kind == "primitive": + if not _same_primitive(left, right): + return False + elif kind == "array": + if len(left) != len(right): + return False + pending.extend(zip(left, right)) + else: + if len(left) != len(right): + return False + for name, value in left.items(): + if name not in right: + return False + pending.append((value, right[name])) + return True + + +def _kind(value: Any) -> str: + if isinstance(value, list): + return "array" + return "object" if isinstance(value, dict) else "primitive" + + +def _same_primitive(left: Any, right: Any) -> bool: + # JSON equality, not Python's: True == 1 and False == 0 in Python. + if isinstance(left, bool) or isinstance(right, bool) or left is None or right is None: + return left is right + if isinstance(left, str) or isinstance(right, str): + return isinstance(left, str) and isinstance(right, str) and left == right + if isinstance(left, (int, float)) and isinstance(right, (int, float)): + # Numbers compare as IEEE doubles, as the other ports parse them, so + # precision beyond a double is not a difference. + return _as_double(left) == _as_double(right) + return False + + +def _as_double(number: int | float) -> float: + # float() rounds an int to nearest and raises exactly when IEEE rounding + # gives an infinity, the value TypeScript parses from the rendered digits. + # math.copysign would raise too: it converts the int to a float first. + try: + return float(number) + except OverflowError: + return math.inf if number > 0 else -math.inf + + +def _utf16_order(name: str) -> bytes: + # Big-endian code units compare as UTF-16 code units, the W02 key order. + return name.encode("utf-16-be", "surrogatepass") + + +def _escape_token(name: str) -> str: + return name.replace("~", "~0").replace("/", "~1") + + +def _run_hook(hook: Callable[[Any, Any], Any], cached: Any, source: Any) -> tuple[list[Entry], int] | None: + """Run the use case's hook under the comparator's synchronous discipline. + + A raise, an awaitable or a result that is not a list returns None. Only + the kept members are read, inside the try, and a path is logged as its + str form: a hook that returns wrong entries gets wrong paths. + """ + try: + result = hook(cached, source) + if inspect.isawaitable(result): + # The contract is synchronous: fail closed without driving it. + discard_awaitable(result) + return None + if not isinstance(result, list): + return None + entries: list[Entry] = [] + for member in result[:MAX_ENTRIES]: + entry: Entry = {"path": str(member["path"])} + for side in ("cached", "source"): + if side in member and member[side] is not UNDEFINED: + entry[side] = member[side] + entries.append(entry) + return entries, len(result) + except (Exception, asyncio.CancelledError): + return None + + +def _render_diff_json(entries: list[Entry]) -> str | None: + # A side with no JSON rendering makes the whole field None, and later + # sides are not rendered. + bounded = [] + for entry in entries: + next_entry = _bound_entry(entry) + if next_entry is None: + return None + bounded.append(next_entry) + return clamp_utf8(_native_json(bounded), DIFF_MAX_BYTES) + + +def _bound_entry(entry: Entry) -> Entry | None: + """Clamp the pointer, and keep each present side as JSON or as its clamped text. + + One clamp rule for built-in and hook entries. None when a side has no JSON + rendering. + """ + bounded: Entry = {"path": clamp_utf8(entry["path"], PATH_MAX_BYTES)} + for side in ("cached", "source"): + if side not in entry: + continue + text = render_json(entry[side]) + if text is None: + return None + clamped = clamp_utf8(text, SIDE_MAX_BYTES) + # Parsing the one rendering keeps each side rendered once. + bounded[side] = json.loads(text) if clamped is text else clamped + return bounded diff --git a/python/dialcache/cache.py b/python/dialcache/cache.py index 6c2dae7b..8bf0dc10 100644 --- a/python/dialcache/cache.py +++ b/python/dialcache/cache.py @@ -8,18 +8,26 @@ from __future__ import annotations import asyncio -import contextvars import functools import inspect -import json import logging import math from collections.abc import Awaitable, Callable, Coroutine, Mapping from dataclasses import dataclass, field from typing import Any, ParamSpec, Protocol, TypeVar, overload +from ._callbacks import discard_awaitable +from ._shadow_log import KEY_MAX_BYTES, MismatchLogPlan, clamp_utf8, render_shadow_mismatch_log from .clock import SystemClock -from .config import UNSET, Policy, _valid_ramp, merge_policy, resolve_layer, validate_static_policy +from .config import ( + UNSET, + Policy, + _valid_ramp, + merge_policy, + resolve_layer, + resolve_mismatch_logging, + validate_static_policy, +) from .context import DialCacheContext from .errors import ( ConfigError, @@ -39,6 +47,7 @@ T = TypeVar("T") P = ParamSpec("P") MAX_SAFE = 9_007_199_254_740_991 +SHADOW_MISMATCH_WARNING = "DialCache shadow validation mismatch: %s" class _CachedDecorator(Protocol): @@ -64,6 +73,15 @@ async def _call_dependency(callback: Callable[..., Any], *args: Any) -> Any: raise RuntimeError("DialCache dependency was cancelled") from error +async def _completes(delivered: Awaitable[Any]) -> bool: + """Await a logger's awaitable; whether it completed without raising.""" + try: + await _call_dependency(lambda: delivered) + except Exception: + return False + return True + + def _valid_integer(value: Any, minimum: int = 0, maximum: int = MAX_SAFE) -> bool: return ( isinstance(value, (int, float)) @@ -146,6 +164,7 @@ class _Operation: tracked: bool comparator: Callable[[Any, Any], bool] recovery: Callable[[BaseException], bool] + mismatch_diff: Callable[[Any, Any], Any] | None = None did_timeout: bool = False deliveries: set[asyncio.Future[Any]] = field(default_factory=set) @@ -165,6 +184,14 @@ class DialCache: The application owns Redis connections. Instances have independent scopes, LRU storage and in-flight tables. Values held in memory are shared by reference and should be treated as immutable. + + ``logger`` receives ``warning(message, details)`` calls and defaults to + the ``dialcache`` standard-library logger. Nothing it raises reaches the + caller, and an awaitable it returns for any warning is driven in a task + of its own. A confirmed shadow-mismatch warning without an earlier + ``hook_error`` or ``render_error`` records its ``shadowMismatchLogging`` + outcome after the call returns, or after that awaitable settles: + ``logger_error`` when either raises. """ def __init__( @@ -265,6 +292,22 @@ def cached( An explicit selector receives the original positional and keyword arguments, including self for methods. Loader defaults are not injected: declare matching defaults on the selector when callers can omit them. + + Options, shared with :meth:`get_or_load`: ``default_config``, + ``fallback_timeout_ms``, ``serializer``, ``track_for_invalidation``, + ``shadow_comparator``, ``shadow_mismatch_diff`` and + ``should_attempt_stale_recovery``. ``shadow_mismatch_diff(cached, + source)`` replaces the built-in JSON difference computation of + confirmed-mismatch warnings, which runs only while + ``shadow.mismatch_logging.diff`` is above ``"off"``. It returns a list + of mappings, each with a str ``"path"`` (an RFC 6901 JSON Pointer) and + optional ``"cached"`` and ``"source"`` sides; a missing key or + ``UNDEFINED`` omits a side, and ``None`` is JSON null. Like the + comparator it must be synchronous, deterministic, side-effect-free, + non-mutating and bounded. A raise, an awaitable, or a result that is + not such a list logs None for the paths and the diff; DialCache never + falls back to the built-in computation. Non-callable callbacks raise + ConfigError. """ if (cache_key is None) == (id_arg is None): raise ConfigError("Supply exactly one of cache_key or id_arg") @@ -322,6 +365,7 @@ def select() -> Any: prototype.tracked, prototype.comparator, prototype.recovery, + prototype.mismatch_diff, ) return await self._execute(op) @@ -339,7 +383,10 @@ async def get_or_load( key_selector: Callable[[], Any] | None = None, **options: Any, ) -> T: - """Read a key or run its loader. Repeated inline use-case names are valid.""" + """Read a key or run its loader. Repeated inline use-case names are valid. + + Options are those of :meth:`cached`, validated on every call. + """ op = self._operation(load, key_selector or (lambda: key), key_type, use_case, **options) return await self._execute(op) @@ -362,6 +409,7 @@ def _operation( track_for_invalidation: bool = False, shadow_comparator: Any = None, should_attempt_stale_recovery: Any = None, + shadow_mismatch_diff: Any = None, ) -> _Operation: self._check_use_case(use_case) policy = validate_static_policy(default_config) or Policy() @@ -371,6 +419,8 @@ def _operation( ) if not callable(comparator) or not callable(recovery): raise ConfigError("Comparator and recovery predicate must be callable") + if shadow_mismatch_diff is not None and not callable(shadow_mismatch_diff): + raise ConfigError("shadow_mismatch_diff must be callable") return _Operation( load, select, @@ -382,6 +432,7 @@ def _operation( track_for_invalidation, comparator, recovery, + shadow_mismatch_diff, ) @staticmethod @@ -404,26 +455,22 @@ def consume(done: asyncio.Task[Any]) -> None: def _emit(self, event: str, labels: Mapping[str, Any], **fields: Any) -> None: emit_metric(self.metrics, {"event": event, **labels, **fields}) - def _log(self, message: str, error: Any = None) -> None: + def _log(self, message: str, *args: Any) -> bool | asyncio.Task[bool]: + """Call ``logger.warning`` so that nothing it does reaches the caller. + + Returns whether the call returned without raising. An awaitable it + returns is driven in a task of its own, returned instead, which + settles to whether the awaitable completed without raising. The + awaitable check reads the result's class, which application code can + make raise, so a raise there is a failed call too. + """ try: - self.logger.warning(message, error) if error is not None else self.logger.warning(message) + delivered = self.logger.warning(message, *args) + if not inspect.isawaitable(delivered): + return True except (Exception, asyncio.CancelledError): - pass - - @staticmethod - def _discard_awaitable(value: Any) -> None: - # Recovery predicates are synchronous. A rejected result must not - # start work or take ownership of an application's existing task. - if inspect.iscoroutine(value): - if inspect.getcoroutinestate(value) == inspect.CORO_CREATED: - value.close() - elif asyncio.isfuture(value): - - def consume(done: asyncio.Future[Any]) -> None: - if not done.cancelled(): - done.exception() - - value.add_done_callback(consume, context=contextvars.Context()) + return False + return self._spawn(_completes(delivered)) def _labels(self, key: Key | _Operation, layer: str | None = None) -> dict[str, Any]: labels = {"cacheNamespace": self.namespace, "useCase": key.use_case, "keyType": key.key_type} @@ -851,7 +898,8 @@ async def _remote_chain(self, op: _Operation, key: Key, policy: Policy, local: A if present: return value else: - self._discard_awaitable(allow) + # Recovery predicates are synchronous. + discard_awaitable(allow) except Exception: pass raise @@ -998,13 +1046,21 @@ def _schedule_shadow( if key.logical in self._shadows or len(self._shadows) >= self._shadow_max: self._emit("shadowValidation", self._labels(key), outcome="dropped") return - log = shadow.get("log_mismatches", shadow.get("logMismatches", False)) - if type(log) is not bool: - self._error(key, "remote", "config_resolution") - log = False + plan = self._resolve_shadow_logging(key, shadow) job = _Shadow(self.clock.monotonic_ms() if started is None else started, op.timeout or 60_000) self._shadows[key.logical] = job - self._spawn(self._run_shadow(op, key, policy, remote, job, frame, source, log)) + self._spawn(self._run_shadow(op, key, policy, remote, job, frame, source, plan)) + + def _resolve_shadow_logging(self, key: Key, shadow: Mapping[str, Any]) -> MismatchLogPlan | None: + """Capture the content leaves at admission; None when both are off.""" + resolved = resolve_mismatch_logging(shadow) + if resolved.invalid: + self._error(key, "remote", "config_resolution") + return ( + MismatchLogPlan(resolved.value, resolved.diff) + if resolved.value or resolved.diff != "off" + else None + ) def _release_shadow(self, key: Key, job: _Shadow) -> None: if job.finished and not job.pending_reads and self._shadows.get(key.logical) is job: @@ -1044,7 +1100,7 @@ async def _run_shadow( job: _Shadow, frame: Frame | None, source: asyncio.Task[Any] | None, - log: bool, + plan: MismatchLogPlan | None, ) -> None: if source is None: try: @@ -1149,12 +1205,10 @@ async def work() -> str: confirmation.payload ) != self._payload_bytes(frame.payload): return "superseded" - if log: - details.update( - cacheKey=self._clamp(key.logical, 2048), - cachedValueJson=self._preview(cached), - sourceValueJson=self._preview(value), - ) + if plan is not None: + # Rendered after the verdict, so warning work can + # neither change it nor hold the capacity slot. + details["compared"] = (cached, value) details["age"] = max(0, self.clock.wall_ms() - frame.created_at_ms) / 1000 return "match" if matched else "mismatch" finally: @@ -1173,30 +1227,58 @@ async def work() -> str: except Exception: outcome = "timeout" self._emit("shadowValidation", self._labels(key), outcome=outcome) + age = None if "age" in details and outcome in ("match", "mismatch"): - self._emit("shadowAge", self._labels(key), outcome=outcome, seconds=details.pop("age")) - if outcome == "mismatch" and log: - self._log( - "DialCache shadow validation mismatch: %s", - {**self._labels(key), "outcome": outcome, **details}, - ) + age = details.pop("age") + self._emit("shadowAge", self._labels(key), outcome=outcome, seconds=age) + if outcome == "mismatch" and plan is not None: + cached, value = details.pop("compared") + self._warn_shadow_mismatch(key, op, plan, cached, value, age) + + def _warn_shadow_mismatch( + self, key: Key, op: _Operation, plan: MismatchLogPlan, cached: Any, source: Any, age: float | None + ) -> None: + """Emit the confirmed-mismatch warning and its one shadowMismatchLogging event. - @staticmethod - def _payload_bytes(value: str | bytes) -> bytes: - return utf8_bytes(value) if isinstance(value, str) else value + Runs after the verdict was recorded and never changes it; any + construction failure logs less, never more, and nothing reaches the + caller. A logger that returns an awaitable settles the event when the + awaitable does, and one that never settles emits none. + """ + warning: dict[str, Any] = { + **self._labels(key), + "outcome": "mismatch", + "cachedValueAgeSeconds": age, + "cacheKey": clamp_utf8(key.logical, KEY_MAX_BYTES), + } + try: + fields, failure = render_shadow_mismatch_log(plan, cached, source, op.mismatch_diff) + except (Exception, asyncio.CancelledError): + # Last resort: an application value whose rendering raises + # CancelledError, or anything unforeseen, still logs the metadata. + fields, failure = {}, "render_error" + warning.update(fields) + if failure is not None: + warning["loggingOutcome"] = failure + delivered = self._log(SHADOW_MISMATCH_WARNING, warning) + labels = self._labels(key) + if failure is not None: + # An earlier failure is recorded at once, whatever the logger does. + self._emit("shadowMismatchLogging", labels, outcome=failure) + elif isinstance(delivered, bool): + self._emit("shadowMismatchLogging", labels, outcome="logged" if delivered else "logger_error") + else: + # The logger's task holds only its awaitable and this callback + # only the labels, so a logger that never settles retains + # neither compared value. + def settled(done: asyncio.Task[bool]) -> None: + if not done.cancelled() and done.exception() is None: + self._emit( + "shadowMismatchLogging", labels, outcome="logged" if done.result() else "logger_error" + ) - @staticmethod - def _clamp(text: str, limit: int) -> str: - encoded = utf8_bytes(text) - marker = b"...[truncated]" - if len(encoded) <= limit: - return text - return encoded[: limit - len(marker)].decode("utf-8", "ignore") + marker.decode() + delivered.add_done_callback(settled) @staticmethod - def _preview(value: Any) -> str | None: - try: - text = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) - return DialCache._clamp(text, 8192) - except Exception: - return None + def _payload_bytes(value: str | bytes) -> bytes: + return utf8_bytes(value) if isinstance(value, str) else value diff --git a/python/dialcache/config.py b/python/dialcache/config.py index ce587183..b88f8e76 100644 --- a/python/dialcache/config.py +++ b/python/dialcache/config.py @@ -78,13 +78,24 @@ def _shadow_map(value: Any) -> Any: result: dict[str, Any] = {} if "ramp" in value: result["ramp"] = value["ramp"] - if "log_mismatches" in value: - result["log_mismatches"] = value["log_mismatches"] - elif "logMismatches" in value: - result["log_mismatches"] = value["logMismatches"] + for native, portable in (("log_mismatches", "logMismatches"), ("mismatch_logging", "mismatchLogging")): + if native in value: + result[native] = value[native] + elif portable in value: + result[native] = value[portable] + if "mismatch_logging" in result: + result["mismatch_logging"] = _mismatch_logging_map(result["mismatch_logging"]) return MappingProxyType(result) +def _mismatch_logging_map(value: Any) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ConfigError("shadow.mismatch_logging must be an object") + # Leaf values stay unvalidated, like layer-map leaves, so a malformed + # runtime leaf fails open narrowly at resolution. + return MappingProxyType({leaf: value[leaf] for leaf in ("value", "diff") if leaf in value}) + + @dataclass(frozen=True) class Policy: """Per-use-case policy; omitted fields inherit in runtime overlays. @@ -94,6 +105,14 @@ class Policy: ramp and optional shadow/recovery leaves remain available for narrow runtime fail-open resolution. Static operation setup uses :func:`validate_static_policy` to reject invalid defaults before calls. + + ``shadow`` takes ``ramp`` and the confirmed-mismatch warning content: + ``mismatch_logging={"value": bool, "diff": "off" | "paths" | "values"}``, + both leaves off when omitted, and ``log_mismatches``, shorthand for + ``{"value": True, "diff": "values"}`` when true and for both leaves off + when false. A leaf written beside the shorthand overrides what it expands + to; a runtime overlay's leaves then win leaf by leaf. The camelCase names + ``mismatchLogging`` and ``logMismatches`` are accepted. """ ttl_sec: Mapping[str, Any] = field(default_factory=dict) @@ -125,7 +144,9 @@ def disabled(cls) -> Policy: return cls( request_local=False, stale_on_error_max_age_sec=0, - shadow={"ramp": 0, "log_mismatches": False}, + # Leaves merge independently, so both are written: an omitted one + # would let an inherited leaf survive disabled(). + shadow={"ramp": 0, "mismatch_logging": {"value": False, "diff": "off"}}, ramp={"local": 0, "remote": 0}, ) @@ -189,16 +210,98 @@ def merge_policy(defaults: PolicyInput, runtime: PolicyInput) -> Policy | None: for name in ("request_local", "coalesce", "stale_on_error_max_age_sec", "remote_read_timeout_ms"): value = getattr(overlay, name) values[name] = getattr(base, name) if value is UNSET else value - if base.shadow is UNSET and overlay.shadow is UNSET: - values["shadow"] = UNSET - else: - values["shadow"] = { - **({} if base.shadow is UNSET else base.shadow), - **({} if overlay.shadow is UNSET else overlay.shadow), - } + values["shadow"] = _merge_shadow(base.shadow, overlay.shadow) return Policy(**values) +def _merge_shadow(base: Any, overlay: Any) -> Any: + if base is UNSET and overlay is UNSET: + return UNSET + defaults: Mapping[str, Any] = {} if base is UNSET else base + runtime: Mapping[str, Any] = {} if overlay is UNSET else overlay + merged: dict[str, Any] = {} + if "ramp" in runtime: + merged["ramp"] = runtime["ramp"] + elif "ramp" in defaults: + merged["ramp"] = defaults["ramp"] + # Each layer's shorthand expands before the leaf-wise merge, so a runtime + # log_mismatches replaces both inherited leaves while a leaf written beside + # it still wins within its own layer. + inherited = expand_mismatch_logging(defaults) + written = expand_mismatch_logging(runtime) + # Only the runtime layer can carry a malformed shorthand: static validation + # rejects one in the operation's defaults. It stays raw, so admission + # records its config_resolution; its expanded leaves already won the merge. + if written.get("invalid"): + merged["log_mismatches"] = runtime["log_mismatches"] + group = { + leaf: written[leaf] if leaf in written else inherited[leaf] + for leaf in ("value", "diff") + if leaf in written or leaf in inherited + } + if group: + merged["mismatch_logging"] = group + return merged + + +MISMATCH_DIFF_LEVELS = ("off", "paths", "values") + + +@dataclass(frozen=True) +class MismatchLogging: + """The effective mismatch-logging leaves and whether any written input was invalid.""" + + value: bool + diff: str + invalid: bool + + +def expand_mismatch_logging(shadow: Mapping[str, Any]) -> dict[str, Any]: + """Expand one shadow group's shorthand into the leaves it stands for. + + ``log_mismatches=True`` expands to ``{"value": True, "diff": "values"}`` and + ``False`` to ``{"value": False, "diff": "off"}``; then the leaves written in + the same group's ``mismatch_logging`` override them. A malformed shorthand + expands to both leaves off and is marked ``invalid``. An omitted leaf stays + omitted, and written leaves pass through unvalidated. + """ + expanded: dict[str, Any] = {} + if "log_mismatches" in shadow: + shorthand = shadow["log_mismatches"] + if shorthand is True: + expanded = {"value": True, "diff": "values"} + elif shorthand is False: + expanded = {"value": False, "diff": "off"} + else: + expanded = {"value": False, "diff": "off", "invalid": True} + if "mismatch_logging" not in shadow: + return expanded + group = shadow["mismatch_logging"] + return {**expanded, **{leaf: group[leaf] for leaf in ("value", "diff") if leaf in group}} + + +def resolve_mismatch_logging(shadow: Mapping[str, Any]) -> MismatchLogging: + """Resolve the effective mismatch-logging leaves of one shadow group. + + Each leaf validates independently: an omitted leaf is off, an invalid leaf + acts off, and ``invalid`` reports that the group owes one config_resolution. + """ + leaves = expand_mismatch_logging(shadow) + value = leaves.get("value", False) + diff = leaves.get("diff", "off") + valid_value = isinstance(value, bool) + valid_diff = _valid_diff_level(diff) + return MismatchLogging( + value=value if valid_value else False, + diff=diff if valid_diff else "off", + invalid=bool(leaves.get("invalid")) or not valid_value or not valid_diff, + ) + + +def _valid_diff_level(value: Any) -> bool: + return isinstance(value, str) and value in MISMATCH_DIFF_LEVELS + + def _valid_ramp(value: Any) -> bool: return ( not isinstance(value, bool) @@ -229,6 +332,11 @@ def validate_static_policy(value: PolicyInput) -> Policy | None: raise ConfigError("shadow.ramp must be a finite number from 0 through 100") if "log_mismatches" in policy.shadow and not isinstance(policy.shadow["log_mismatches"], bool): raise ConfigError("shadow.log_mismatches must be a boolean") + mismatch_logging = policy.shadow.get("mismatch_logging", {}) + if "value" in mismatch_logging and not isinstance(mismatch_logging["value"], bool): + raise ConfigError("shadow.mismatch_logging.value must be a boolean") + if "diff" in mismatch_logging and not _valid_diff_level(mismatch_logging["diff"]): + raise ConfigError('shadow.mismatch_logging.diff must be "off", "paths" or "values"') return policy diff --git a/python/dialcache/serializer.py b/python/dialcache/serializer.py index 4e8e8ef2..89a67341 100644 --- a/python/dialcache/serializer.py +++ b/python/dialcache/serializer.py @@ -38,11 +38,7 @@ class JsonSerializer: def dump(self, value: object) -> str: if value is UNDEFINED: return JSON_UNDEFINED_SENTINEL - payload = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) - # JSON must escape lone surrogate units before the frame's UTF-8 text - # boundary replaces malformed text. This preserves JSON string values - # in the same way as well-formed ECMAScript JSON.stringify. - return payload.encode("utf-8", errors="backslashreplace").decode("utf-8") + return _native_json(value) def load(self, value: Payload) -> object: payload = value.decode("utf-8", errors="replace") if isinstance(value, bytes) else value @@ -53,3 +49,15 @@ def load(self, value: Payload) -> object: @staticmethod def _reject_constant(value: str) -> object: raise ValueError(f"Invalid JSON constant: {value}") + + +def _native_json(value: object) -> str: + """Compact native JSON with lone surrogate units escaped; raises for values outside JSON. + + The one rendering both stored payloads and shadow-mismatch warnings use. + JSON must escape lone surrogate units before a UTF-8 text boundary + replaces malformed text. This preserves JSON string values in the same way + as well-formed ECMAScript JSON.stringify. + """ + payload = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + return payload.encode("utf-8", errors="backslashreplace").decode("utf-8") diff --git a/python/tests/formal/driver.py b/python/tests/formal/driver.py index 87d4d506..9edcfdde 100644 --- a/python/tests/formal/driver.py +++ b/python/tests/formal/driver.py @@ -9,7 +9,7 @@ import math from dataclasses import dataclass -from dialcache import DialCache +from dialcache import UNDEFINED, DialCache from dialcache.errors import FallbackTimeoutError, MissingRemoteError from dialcache.key import Key from dialcache.local import LocalCache @@ -17,7 +17,23 @@ from .executor import WALL_EPOCH_MS, Executor -ABSENT = object() +# The pending adapter reply when none is queued; never a cache value. +NO_REPLY = object() + + +class Unrenderable: + """The source result with no JSON rendering (D13). + + json.dumps rejects it, so every rendering of it fails, and the default + comparator finds it unequal to every decoded value. The driver knows it by + identity and observes a call that returns it as {"unrenderable": true}. + """ + + def __repr__(self): + return "UNRENDERABLE" + + +UNRENDERABLE = Unrenderable() def empty_observation(fixture=None): @@ -44,11 +60,28 @@ def empty_observation(fixture=None): def json_value(value): - return {"absent": True} if value is ABSENT else value + if value is UNDEFINED: + return {"absent": True} + return {"unrenderable": True} if value is UNRENDERABLE else value def dump_value(value): - return "undefined" if value is ABSENT else json.dumps(value, ensure_ascii=False, separators=(",", ":")) + return "undefined" if value is UNDEFINED else json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def warning_observation(details): + """The warning's fields as logged, each present exactly when the warning carries it. + + The mismatchPaths list crosses the wire as one JSON string, so every + observed field stays a primitive (PORTING.md, mismatchWarning). + """ + fields = dict(details) + if "mismatchPaths" in fields: + paths = fields.pop("mismatchPaths") + fields["mismatchPathsJson"] = ( + None if paths is None else json.dumps(paths, ensure_ascii=False, separators=(",", ":")) + ) + return fields class FakeRedis: @@ -100,8 +133,8 @@ async def abort(): await owner.hold("read", index) if owner.faults.get("read"): raise RuntimeError("Controlled read failure") - if owner.adapter_reply is not ABSENT: - result, owner.adapter_reply = owner.adapter_reply, ABSENT + if owner.adapter_reply is not NO_REPLY: + result, owner.adapter_reply = owner.adapter_reply, NO_REPLY # Bind the coordinator's JSON record to the native adapter # variants. Keep malformed reason/timestamp leaves intact so # the production trust boundary, not the driver, validates them. @@ -189,12 +222,22 @@ def warning(self, *args, **kwargs): and args[0] == "DialCache shadow validation mismatch: %s" and isinstance(args[1], dict) ): - self.owner.record("mismatchWarning", **args[1]) + self.owner.record("mismatchWarning", **warning_observation(args[1])) + self.observer_failure() + # The logger fault fails every warning call after recording it; + # metric observers keep working. + if self.owner.faults.get("logger"): + raise RuntimeError("Controlled logger failure") + + def debug(self, *args, **kwargs): + self.observer_failure() + + error = debug + + def observer_failure(self): if self.owner.fixture.get("observerFailure") or self.owner.faults.get("observer"): raise RuntimeError("Controlled observer failure") - debug = error = warning - class Serializer: def __init__(self, owner): @@ -215,7 +258,7 @@ async def load(self, raw): if self.owner.faults.get("load"): raise RuntimeError("Controlled deserialization failure") text = raw.decode() if isinstance(raw, bytes) else raw - return ABSENT if text == "undefined" else json.loads(text) + return UNDEFINED if text == "undefined" else json.loads(text) class FaultingLocal: @@ -258,7 +301,7 @@ def __init__(self, fixture, *, settle=True): self.scopes = {} self.instances = {} self.runtime_policy = {} - self.adapter_reply = ABSENT + self.adapter_reply = NO_REPLY self.maintenance_error = RuntimeError("Controlled mutation failure") self.redis = FakeRedis(self.clock, self) self.cache = self.instance("default") @@ -347,6 +390,9 @@ def compare(*args, **kwargs): raise RuntimeError("Controlled comparison failure") return self.fixture["comparator"] == "equal" + def mismatch_diff(*args, **kwargs): + raise RuntimeError("Controlled mismatch diff failure") + options = { "key_type": "id", "key": command.get("key", "1"), @@ -359,6 +405,8 @@ def compare(*args, **kwargs): options["should_attempt_stale_recovery"] = self.classifier(command["recovery"]) if "comparator" in self.fixture: options["shadow_comparator"] = compare + if "mismatchDiff" in self.fixture: + options["shadow_mismatch_diff"] = mismatch_diff if self.fixture.get("fallbackTimeoutMs") != "default": options["fallback_timeout_ms"] = self.fixture.get("fallbackTimeoutMs", 10) @@ -407,7 +455,8 @@ def apply(self, command): if op == "begin": self.begin(command) elif op == "resolve": - self.loaders[command["loader"]].set_result(command.get("value", ABSENT)) + value = UNRENDERABLE if command.get("unrenderable") else command.get("value", UNDEFINED) + self.loaders[command["loader"]].set_result(value) elif op == "reject": index = command["loader"] if command.get("error") == "timeout": @@ -422,7 +471,7 @@ def apply(self, command): elif op == "faults": self.faults.update(command["value"]) elif op == "adapterReply": - if self.adapter_reply is not ABSENT: + if self.adapter_reply is not NO_REPLY: raise RuntimeError("Unconsumed adapter reply") self.adapter_reply = command["value"] elif op == "release": @@ -438,7 +487,7 @@ def apply(self, command): payload = ( bytes.fromhex(command["payloadHex"]) if "payloadHex" in command - else command.get("payloadText", dump_value(command.get("value", ABSENT))) + else command.get("payloadText", dump_value(command.get("value", UNDEFINED))) ) frame = encode_frame(payload, self.clock.wall_ms() - command.get("ageMs", 0)) self.redis.seed(self.value_key(command).value_key, frame, command.get("ttlMs", 60_000)) diff --git a/python/tests/test_config.py b/python/tests/test_config.py index 1b0f6040..da446395 100644 --- a/python/tests/test_config.py +++ b/python/tests/test_config.py @@ -6,11 +6,14 @@ MAX_CACHE_TTL_SEC, MAX_TIMER_DELAY_MS, UNSET, + MismatchLogging, Policy, deterministic_ramp_sample, + expand_mismatch_logging, merge_policy, normalize_policy, resolve_layer, + resolve_mismatch_logging, validate_deadline_ms, validate_static_policy, ) @@ -36,7 +39,8 @@ def test_sparse_merge_snapshots_every_leaf_and_inherits_provider_none(): assert merged.coalesce is False assert merged.remote_read_timeout_ms == 75 assert merged.stale_on_error_max_age_sec == 120 - assert merged.shadow == {"ramp": 0, "log_mismatches": True} + # The default shorthand expands before the merge, so only leaves remain. + assert merged.shadow == {"ramp": 0, "mismatch_logging": {"value": True, "diff": "values"}} assert merge_policy(defaults, None) is defaults assert merge_policy(None, None) is None with pytest.raises(TypeError): @@ -105,7 +109,8 @@ def test_disabled_overlay_turns_off_all_inherited_paths(): merged = merge_policy(default, Policy.disabled()) assert merged.request_local is False assert merged.stale_on_error_max_age_sec == 0 - assert merged.shadow == {"ramp": 0, "log_mismatches": False} + assert merged.shadow == {"ramp": 0, "mismatch_logging": {"value": False, "diff": "off"}} + assert resolve_mismatch_logging(merged.shadow) == MismatchLogging(False, "off", False) assert merged.coalesce is UNSET assert resolve_layer(merged, "key", "local").reason == "ramped_down" assert resolve_layer(merged, "key", "remote").reason == "ramped_down" @@ -140,3 +145,260 @@ def test_deadline_and_ttl_boundaries_are_inclusive(): assert validate_deadline_ms(MAX_TIMER_DELAY_MS) == MAX_TIMER_DELAY_MS policy = validate_static_policy(Policy.enabled(MAX_CACHE_TTL_SEC)) assert resolve_layer(policy, "key", "local").ttl_sec == MAX_CACHE_TTL_SEC + + +ABSENT = object() + + +def shadow_layer(shorthand, group, camel=False): + """One shadow group with each field omitted (ABSENT) or written.""" + shadow = {} + if shorthand is not ABSENT: + shadow["logMismatches" if camel else "log_mismatches"] = shorthand + if group is not ABSENT: + shadow["mismatchLogging" if camel else "mismatch_logging"] = group + return shadow + + +def expected_leaves(default, runtime): + """The recorded rule, written out without the code under test. + + A written shorthand stands for both leaves (True: value on and diff + "values"; False or malformed: both off), a leaf written beside it wins + within its own layer, the runtime layer wins per leaf, an omitted leaf is + off, an invalid leaf acts off, and an invalid input owes one + config_resolution. Defaults stay valid, as static validation guarantees. + """ + + def leaves(shorthand, group): + result = {} + if shorthand is not ABSENT: + result = {"value": shorthand is True, "diff": "values" if shorthand is True else "off"} + if group is not ABSENT: + result.update({leaf: group[leaf] for leaf in ("value", "diff") if leaf in group}) + return result + + inherited, written = leaves(*default), leaves(*runtime) + value = written["value"] if "value" in written else inherited.get("value", ABSENT) + diff = written["diff"] if "diff" in written else inherited.get("diff", ABSENT) + valid_value = value is ABSENT or isinstance(value, bool) + valid_diff = diff is ABSENT or (isinstance(diff, str) and diff in ("off", "paths", "values")) + malformed = runtime[0] is not ABSENT and not isinstance(runtime[0], bool) + return MismatchLogging( + value=valid_value and value is True, + diff=diff if valid_diff and diff is not ABSENT else "off", + invalid=malformed or not valid_value or not valid_diff, + ) + + +def test_merge_expands_each_layer_before_leaves_merge_for_every_combination(): + default_shorthands = [ABSENT, True, False] + default_groups = [ + ABSENT, + {}, + {"value": True}, + {"value": False}, + {"diff": "paths"}, + {"diff": "off"}, + {"value": True, "diff": "values"}, + ] + runtime_shorthands = [ABSENT, True, False, "yes", None, 0, 1] + runtime_groups = [ + ABSENT, + {"value": True}, + {"value": False}, + {"diff": "paths"}, + {"diff": "values"}, + {"value": True, "diff": "off"}, + {"value": "yes"}, + {"value": None}, + {"diff": "full"}, + {"diff": "toString"}, + {"diff": None}, + {"unknown": True}, + ] + combinations = 0 + for default_shorthand in default_shorthands: + for default_group in default_groups: + default = validate_static_policy(Policy(shadow=shadow_layer(default_shorthand, default_group))) + for runtime_shorthand in runtime_shorthands: + for runtime_group in runtime_groups: + expected = expected_leaves( + (default_shorthand, default_group), (runtime_shorthand, runtime_group) + ) + for camel in (False, True): + runtime = {"shadow": shadow_layer(runtime_shorthand, runtime_group, camel)} + merged = merge_policy(default, runtime) + assert resolve_mismatch_logging(merged.shadow) == expected, ( + default.shadow, + runtime, + ) + combinations += 1 + assert combinations == 3 * 7 * 7 * 12 * 2 + + # disabled() on either side, and a runtime policy without a shadow group. + disabled = Policy.disabled().shadow + for default, runtime, expected in [ + ({"log_mismatches": True}, Policy.disabled(), MismatchLogging(False, "off", False)), + ( + disabled, + {"shadow": {"mismatchLogging": {"diff": "paths"}}}, + MismatchLogging(False, "paths", False), + ), + ( + {"log_mismatches": True, "mismatch_logging": {"diff": "paths"}}, + {"ttl_sec": {}}, + MismatchLogging(True, "paths", False), + ), + ]: + merged = merge_policy(Policy(shadow=default), runtime) + assert resolve_mismatch_logging(merged.shadow) == expected + + +@pytest.mark.parametrize( + ("default", "runtime", "effective"), + [ + (None, {"log_mismatches": True, "mismatch_logging": {"value": False}}, (False, "values")), + ({"log_mismatches": True, "mismatch_logging": {"value": False}}, None, (False, "values")), + (None, {"log_mismatches": False, "mismatch_logging": {"diff": "paths"}}, (False, "paths")), + ({"log_mismatches": False, "mismatch_logging": {"diff": "paths"}}, None, (False, "paths")), + ({"mismatch_logging": {"diff": "paths"}}, {"log_mismatches": False}, (False, "off")), + ({"mismatch_logging": {"value": False}}, {"log_mismatches": True}, (True, "values")), + ], +) +@pytest.mark.parametrize("provider_none", [False, True]) +def test_recorded_shorthand_table(default, runtime, effective, provider_none): + defaults = validate_static_policy(Policy(shadow=default) if default is not None else Policy()) + if provider_none and runtime is not None: + return + merged = merge_policy(defaults, None if provider_none else {"shadow": runtime or {}}) + assert resolve_mismatch_logging(merged.shadow) == MismatchLogging(*effective, False) + + +def test_runtime_leaves_inherit_and_override_default_leaves(): + merged = merge_policy( + Policy(shadow={"mismatch_logging": {"value": True, "diff": "paths"}}), + {"shadow": {"mismatch_logging": {"diff": "values"}}}, + ) + assert merged.shadow == {"mismatch_logging": {"value": True, "diff": "values"}} + merged = merge_policy( + Policy(shadow={"mismatch_logging": {"value": True, "diff": "values"}}), + {"shadow": {"mismatch_logging": {"value": False}}}, + ) + assert merged.shadow == {"mismatch_logging": {"value": False, "diff": "values"}} + # A default shorthand is a baseline like any other pair of default leaves. + merged = merge_policy( + Policy(shadow={"log_mismatches": True}), {"shadow": {"mismatchLogging": {"diff": "paths"}}} + ) + assert merged.shadow == {"mismatch_logging": {"value": True, "diff": "paths"}} + # A runtime shorthand replaces both inherited leaves. + merged = merge_policy( + Policy(shadow={"ramp": 5, "mismatch_logging": {"value": True, "diff": "paths"}}), + {"shadow": {"log_mismatches": False}}, + ) + assert merged.shadow == {"ramp": 5, "mismatch_logging": {"value": False, "diff": "off"}} + + +def test_merge_keeps_a_malformed_runtime_shorthand_for_admission(): + merged = merge_policy( + Policy(shadow={"mismatch_logging": {"value": False, "diff": "values"}}), + {"shadow": {"logMismatches": "yes", "mismatchLogging": {"value": True}}}, + ) + # The malformed shorthand expands to both leaves off over the default, the + # leaf written beside it still wins, and the raw value stays so admission + # records its config_resolution. + assert merged.shadow == {"log_mismatches": "yes", "mismatch_logging": {"value": True, "diff": "off"}} + assert resolve_mismatch_logging(merged.shadow) == MismatchLogging(True, "off", True) + + +@pytest.mark.parametrize( + ("shadow", "expected"), + [ + ({}, (False, "off", False)), + ({"mismatch_logging": {}}, (False, "off", False)), + ({"mismatch_logging": {"value": True, "diff": "paths"}}, (True, "paths", False)), + ({"mismatch_logging": {"value": "true", "diff": "paths"}}, (False, "paths", True)), + ({"mismatch_logging": {"value": None}}, (False, "off", True)), + ({"mismatch_logging": {"value": 1}}, (False, "off", True)), + ({"mismatch_logging": {"value": True, "diff": "full"}}, (True, "off", True)), + ({"mismatch_logging": {"diff": 1}}, (False, "off", True)), + ({"mismatch_logging": {"diff": None}}, (False, "off", True)), + ({"mismatch_logging": {"diff": ["paths"]}}, (False, "off", True)), + ({"mismatch_logging": {"value": True, "diff": "__class__"}}, (True, "off", True)), + ({"log_mismatches": "yes"}, (False, "off", True)), + ({"log_mismatches": None}, (False, "off", True)), + ({"log_mismatches": 1}, (False, "off", True)), + ( + {"log_mismatches": "yes", "mismatch_logging": {"value": True, "diff": "values"}}, + (True, "values", True), + ), + ], +) +def test_resolves_each_leaf_independently(shadow, expected): + assert resolve_mismatch_logging(shadow) == MismatchLogging(*expected) + + +def test_expands_the_shorthand_before_the_same_group_leaves(): + assert expand_mismatch_logging({}) == {} + assert expand_mismatch_logging({"log_mismatches": True}) == {"value": True, "diff": "values"} + assert expand_mismatch_logging({"log_mismatches": False}) == {"value": False, "diff": "off"} + assert expand_mismatch_logging({"log_mismatches": "yes"}) == { + "value": False, + "diff": "off", + "invalid": True, + } + assert expand_mismatch_logging({"log_mismatches": True, "mismatch_logging": {"value": False}}) == { + "value": False, + "diff": "values", + } + assert expand_mismatch_logging( + {"log_mismatches": "yes", "mismatch_logging": {"value": True, "diff": "paths"}} + ) == {"value": True, "diff": "paths", "invalid": True} + # Written leaves pass through unvalidated: resolution owns their validation. + assert expand_mismatch_logging({"mismatch_logging": {"diff": "full"}}) == {"diff": "full"} + + +@pytest.mark.parametrize("bad", [None, [], 1, "paths"]) +@pytest.mark.parametrize("name", ["mismatch_logging", "mismatchLogging"]) +def test_non_object_mismatch_logging_fails_resolution(name, bad): + with pytest.raises(ConfigError, match="mismatch_logging must be an object"): + Policy(shadow={name: bad}) + with pytest.raises(ConfigError, match="mismatch_logging must be an object"): + merge_policy(Policy.enabled(10), {"shadow": {name: bad}}) + + +def test_mismatch_logging_snapshots_known_leaves_and_accepts_camel_case(): + group = {"value": True, "diff": "paths", "key": True} + policy = Policy(shadow={"logMismatches": False, "mismatchLogging": group}) + group["diff"] = "values" + assert policy.shadow == {"log_mismatches": False, "mismatch_logging": {"value": True, "diff": "paths"}} + with pytest.raises(TypeError): + policy.shadow["mismatch_logging"]["diff"] = "values" + snake_wins = Policy(shadow={"mismatch_logging": {"value": True}, "mismatchLogging": {"value": False}}) + assert snake_wins.shadow == {"mismatch_logging": {"value": True}} + + +@pytest.mark.parametrize( + "shadow", + [ + {"log_mismatches": "yes"}, + {"log_mismatches": None}, + {"mismatch_logging": {"value": "true"}}, + {"mismatch_logging": {"value": None}}, + {"mismatch_logging": {"diff": "full"}}, + {"mismatch_logging": {"diff": 1}}, + {"mismatch_logging": {"diff": None}}, + ], +) +def test_static_policy_rejects_invalid_mismatch_logging(shadow): + with pytest.raises(ConfigError): + validate_static_policy(Policy(shadow=shadow)) + + +def test_static_policy_accepts_valid_mismatch_logging(): + for shadow in [ + {"mismatch_logging": {"value": True, "diff": "paths"}}, + {"logMismatches": False, "mismatchLogging": {"diff": "values"}}, + {"log_mismatches": True, "mismatch_logging": {"value": False, "diff": "off"}}, + ]: + assert validate_static_policy(Policy(shadow=shadow)).shadow is not UNSET diff --git a/python/tests/test_observer_recovery_boundaries.py b/python/tests/test_observer_recovery_boundaries.py index 21dccd59..c0669921 100644 --- a/python/tests/test_observer_recovery_boundaries.py +++ b/python/tests/test_observer_recovery_boundaries.py @@ -79,6 +79,91 @@ def read(self, request, context): assert warnings == [] +@pytest.mark.parametrize("logger_fails", [False, True], ids=["completes", "raises"]) +def test_drives_an_async_logger_for_a_warning_outside_shadow_validation(executor, logger_fails): + # One logger boundary owns every warning's awaitable, so a coroutine + # logger runs instead of being dropped unawaited, and its failure reaches + # neither the caller nor the loop's unhandled-exception handler. + logged = [] + config_error = RuntimeError("policy unavailable") + + class Logger: + async def warning(self, *args): + await asyncio.sleep(0) + logged.append(args) + if logger_fails: + raise RuntimeError("logger transport unavailable") + + def provider(key): + raise config_error + + cache = DialCache(clock=executor.clock, policy_provider=provider, logger=Logger()) + with cache.enable(): + result = executor.finish( + cache.get_or_load( + lambda: "source", + key="entity-id", + key_type="entity", + use_case="AsyncLogger", + default_config=Policy(ttl_sec={"local": 10}), + ) + ) + executor.drain() + + assert result == "source" + assert logged == [("Could not resolve DialCache policy: %s", config_error)] + + +@pytest.mark.parametrize("warning", ["key_construction", "config_resolution"]) +def test_a_logger_result_whose_class_raises_never_reaches_the_caller(executor, warning): + # The awaitable check reads the result's class, so it runs inside the + # logger boundary: a raise there is a failed call, not the caller's error. + logged = [] + failure = RuntimeError("dependency unavailable") + + class Uninspectable: + @property + def __class__(self): + raise RuntimeError("logger result unavailable") + + class Logger: + def warning(self, *args): + logged.append(args) + return Uninspectable() + + def provider(key): + raise failure + + def select_key(): + raise failure + + cache = DialCache( + clock=executor.clock, + policy_provider=provider if warning == "config_resolution" else None, + logger=Logger(), + ) + + @cache.cached( + key_type="entity", + cache_key=select_key if warning == "key_construction" else (lambda: "entity-id"), + use_case="UninspectableLogger", + default_config=Policy(ttl_sec={"local": 10}), + ) + def load(): + return "source" + + with cache.enable(): + result = executor.finish(load()) + executor.drain() + + message = { + "key_construction": "Could not construct DialCache key: %s", + "config_resolution": "Could not resolve DialCache policy: %s", + }[warning] + assert result == "source" + assert logged == [(message, failure)] + + def failing_recovery_call(executor, cache, failure): def source(): raise failure diff --git a/python/tests/test_protocol_vectors.py b/python/tests/test_protocol_vectors.py index 1f2b1620..2778fcfb 100644 --- a/python/tests/test_protocol_vectors.py +++ b/python/tests/test_protocol_vectors.py @@ -8,7 +8,9 @@ import pytest import zstandard +from formal.schema import json_equal, strict_json +from dialcache._shadow_log import MismatchLogPlan, diff_shadow_mismatch_json, render_shadow_mismatch_log from dialcache.key import Key, normalize_args, ramp_hash, ramp_sample from dialcache.protocol import ( Frame, @@ -59,7 +61,7 @@ def key_from(vector): def test_schemas_provenance_and_inventory(): assert all(corpus["schemaVersion"] == 3 for corpus in CORPORA) - expected_counts = [134, 457, 589, 297] + expected_counts = [191, 457, 589, 297] for corpus, expected in zip(CORPORA, expected_counts, strict=True): rows = [row for value in corpus.values() if isinstance(value, list) for row in value] assert len(rows) == expected @@ -203,6 +205,32 @@ def test_compression_selection(vector): assert decompress_payload(result.payload).payload == raw +@pytest.mark.parametrize("vector", vectors("mismatchPathVectors")) +def test_mismatch_paths(vector): + # W10: the built-in walk over two JSON values, its entries compared as + # parsed JSON. Both projections of one entry list are checked: the paths + # and count at diff "paths", and the entries with their values at "values". + # An omitted cached or source member is an absent side (D13), UNDEFINED. + cached = vector.get("cached", UNDEFINED) + source = vector.get("source", UNDEFINED) + if cached is not UNDEFINED and source is not UNDEFINED: + entries, count = diff_shadow_mismatch_json(cached, source) + assert json_equal(entries, vector["entries"]), (entries, vector["entries"]) + assert count == vector["count"] + paths = [entry["path"] for entry in vector["entries"]] + fields, failure = render_shadow_mismatch_log(MismatchLogPlan(value=False, diff="values"), cached, source) + assert failure is None + assert fields.keys() == {"mismatchPaths", "mismatchPathCount", "diffJson"} + assert fields["mismatchPaths"] == paths + assert fields["mismatchPathCount"] == vector["count"] + diff = strict_json(fields["diffJson"]) + assert json_equal(diff, vector["entries"]), (diff, vector["entries"]) + assert render_shadow_mismatch_log(MismatchLogPlan(value=False, diff="paths"), cached, source) == ( + {"mismatchPaths": paths, "mismatchPathCount": vector["count"]}, + None, + ) + + def assert_wire_vector(group, vector): """Completion reporters call these same real assertions before granting credit.""" assertions = { @@ -218,6 +246,7 @@ def assert_wire_vector(group, vector): "envelopeVectors": test_envelopes, "compressedDecodeVectors": test_compressed_decode, "compressionWriteVectors": test_compression_selection, + "mismatchPathVectors": test_mismatch_paths, } try: assertion = assertions[group] diff --git a/python/tests/test_shadow_log.py b/python/tests/test_shadow_log.py new file mode 100644 index 00000000..ad137421 --- /dev/null +++ b/python/tests/test_shadow_log.py @@ -0,0 +1,826 @@ +"""The built-in mismatch walk, its caps and rendering, and the shadow_mismatch_diff hook.""" + +from __future__ import annotations + +import asyncio +import gc +import json +import math +import tracemalloc +import warnings +from pathlib import Path +from types import MappingProxyType + +import pytest +from formal.schema import json_equal + +from dialcache import UNDEFINED, JsonSerializer, _shadow_log +from dialcache._shadow_log import ( + KEY_MAX_BYTES, + TRUNCATION_MARKER, + VALUE_MAX_BYTES, + MismatchLogPlan, + clamp_utf8, + diff_shadow_mismatch_json, + render_json, + render_shadow_mismatch_log, +) +from dialcache.serializer import JSON_UNDEFINED_SENTINEL + +VALUE_ONLY = MismatchLogPlan(value=True, diff="off") +PATHS_ONLY = MismatchLogPlan(value=False, diff="paths") +VALUES_ONLY = MismatchLogPlan(value=False, diff="values") +EVERYTHING = MismatchLogPlan(value=True, diff="values") + +# The built-in walk's portable vectors (W10), read from the shared corpus every +# port replays; they also pin the examples docs/shadow-validation.md shows. An +# omitted cached or source member is an absent side (D13), UNDEFINED here. +PROTOCOL_VECTORS = Path(__file__).resolve().parents[2] / "formal/catalogs/protocol-vectors.json" +MISMATCH_PATH_VECTORS = json.loads(PROTOCOL_VECTORS.read_text(encoding="utf-8"))["mismatchPathVectors"] +PRESENT_MISMATCH_PATH_VECTORS = [ + vector for vector in MISMATCH_PATH_VECTORS if "cached" in vector and "source" in vector +] + +CACHED_USER = {"id": "123", "email": "old@example.com"} +SOURCE_USER = {"id": "123", "email": "new@example.com"} +CACHED_USER_JSON = '{"id":"123","email":"old@example.com"}' +SOURCE_USER_JSON = '{"id":"123","email":"new@example.com"}' +FAILED_DIFF = {"mismatchPaths": None, "mismatchPathCount": None, "diffJson": None} + + +def circular(): + value = {} + value["self"] = value + return value + + +def nest_members(depth, leaf): + value = leaf + for _ in range(depth): + value = {"n": value} + return value + + +class Stateful(dict): + """The native analog of a stateful toJSON: the encoder calls items() per rendering.""" + + def __init__(self, email): + # Non-empty: the encoder renders an empty dict without calling items(). + super().__init__(email=email) + self.email = email + self.renders = 0 + + def items(self): + self.renders += 1 + return [("email", self.email), ("render", self.renders)] + + +def assert_entries(actual, expected): + assert json_equal(actual, expected), (actual, expected) + # Each entry keeps the path, cached, source order diffJson renders. + assert [list(entry) for entry in actual] == [list(entry) for entry in expected] + + +def test_renders_native_json_for_the_compared_values(): + assert clamp_utf8("urn:user_id:123#GetUser", KEY_MAX_BYTES) == "urn:user_id:123#GetUser" + assert render_shadow_mismatch_log( + VALUE_ONLY, + {"id": "123", "name": "Zoë", "tags": ("a", "b")}, + {"id": "123", "name": "Zoë", "tags": ["a", "c"]}, + ) == ( + { + "cachedValueJson": '{"id":"123","name":"Zoë","tags":["a","b"]}', + "sourceValueJson": '{"id":"123","name":"Zoë","tags":["a","c"]}', + }, + None, + ) + + +def test_escapes_lone_surrogates_as_ecmascript_json_does(): + assert render_json("\ud800") == '"\\ud800"' + assert render_json({"\udfff": 1}) == '{"\\udfff":1}' + fields, failure = render_shadow_mismatch_log(EVERYTHING, {"k": "\ud800"}, {"k": "x"}) + assert failure is None + assert fields["cachedValueJson"] == '{"k":"\\ud800"}' + assert fields["diffJson"] == '[{"path":"/k","cached":"\\ud800","source":"x"}]' + + +def test_renders_what_json_serializer_stores_except_the_undefined_sentinel(): + serializer = JsonSerializer() + for value in [None, True, 1.5, "雪\ud800", {"k": [1, None, ("t",)]}]: + assert render_json(value) == serializer.dump(value) + # A negative-cache entry stores the sentinel; as a compared side it is absent. + assert serializer.dump(UNDEFINED) == JSON_UNDEFINED_SENTINEL + assert render_json(UNDEFINED) is None + for value in [math.nan, object(), circular()]: + assert render_json(value) is None + + +def test_nulls_each_unrenderable_side_independently(): + assert render_shadow_mismatch_log(VALUE_ONLY, circular(), {"version": 2}) == ( + {"cachedValueJson": None, "sourceValueJson": '{"version":2}'}, + "render_error", + ) + assert render_shadow_mismatch_log(VALUE_ONLY, object(), UNDEFINED) == ( + {"cachedValueJson": None, "sourceValueJson": None}, + "render_error", + ) + + +def test_byte_clamps_keys_and_json_without_splitting_utf8(): + key = "k" * KEY_MAX_BYTES + "🙂" + value = {"text": "🙂" * VALUE_MAX_BYTES} + key_preview = clamp_utf8(key, KEY_MAX_BYTES) + fields, failure = render_shadow_mismatch_log(VALUE_ONLY, value, value) + preview = fields["cachedValueJson"] + + assert failure is None + assert len(key_preview.encode()) <= KEY_MAX_BYTES + assert len(preview.encode()) <= VALUE_MAX_BYTES + assert key_preview.endswith(TRUNCATION_MARKER) + assert preview.endswith(TRUNCATION_MARKER) + assert "�" not in key_preview + preview + assert fields["sourceValueJson"] == preview + + +@pytest.mark.parametrize("vector", PRESENT_MISMATCH_PATH_VECTORS, ids=lambda vector: vector["name"]) +def test_computes_the_expected_entries(vector): + entries, count = diff_shadow_mismatch_json(vector["cached"], vector["source"]) + assert_entries(entries, vector["entries"]) + assert count == vector["count"] + + +# An absent side reaches the walk only through rendering, as UNDEFINED. +@pytest.mark.parametrize("vector", MISMATCH_PATH_VECTORS, ids=lambda vector: vector["name"]) +def test_renders_the_expected_diff_fields(vector): + cached, source = vector.get("cached", UNDEFINED), vector.get("source", UNDEFINED) + fields, failure = render_shadow_mismatch_log(VALUES_ONLY, cached, source) + + assert failure is None + assert fields["mismatchPaths"] == [entry["path"] for entry in vector["entries"]] + assert fields["mismatchPathCount"] == vector["count"] + assert_entries(json.loads(fields["diffJson"]), vector["entries"]) + assert render_shadow_mismatch_log(PATHS_ONLY, cached, source) == ( + {"mismatchPaths": fields["mismatchPaths"], "mismatchPathCount": vector["count"]}, + None, + ) + + +@pytest.mark.parametrize( + ("cached", "source", "entries"), + [ + (True, 1, [{"path": "", "cached": True, "source": 1}]), + (0, False, [{"path": "", "cached": 0, "source": False}]), + ( + {"a": [True, 0]}, + {"a": [1, False]}, + [ + {"path": "/a/0", "cached": True, "source": 1}, + {"path": "/a/1", "cached": 0, "source": False}, + ], + ), + (None, False, [{"path": "", "cached": None, "source": False}]), + ("1", 1, [{"path": "", "cached": "1", "source": 1}]), + ], +) +def test_compares_json_primitives_by_json_type(cached, source, entries): + # Python's True == 1 and False == 0 must not hide a JSON difference. + actual, count = diff_shadow_mismatch_json(cached, source) + assert_entries(actual, entries) + assert count == len(entries) + + +@pytest.mark.parametrize( + ("cached", "source", "entries"), + [ + ('{"n":1}', '{"n":1.0}', []), + ("true", "1", [{"path": "", "cached": True, "source": 1}]), + ("false", "0", [{"path": "", "cached": False, "source": 0}]), + ("0", "-0", []), + ("0", "-0.0", []), + ('{"id":9007199254740993}', '{"id":9007199254740992}', []), + ], + ids=[ + "int-and-float", + "true-and-1", + "false-and-0", + "zero-and-negative-zero", + "zero-and-float-negative-zero", + "beyond-double-precision", + ], +) +def test_compares_numbers_as_doubles_and_never_as_bools(cached, source, entries): + # The number rule the other ports follow by parsing JSON numbers as + # doubles: precision beyond a double is not a difference, and a bool is + # never a number. + cached, source = json.loads(cached), json.loads(source) + + actual, count = diff_shadow_mismatch_json(cached, source) + assert_entries(actual, entries) + assert count == len(entries) + fields, failure = render_shadow_mismatch_log(VALUES_ONLY, cached, source) + assert failure is None + assert fields["mismatchPaths"] == [entry["path"] for entry in entries] + assert fields["mismatchPathCount"] == len(entries) + assert_entries(json.loads(fields["diffJson"]), entries) + + +def test_compares_integers_beyond_the_double_range_as_doubles(): + # Only Python renders such integers. TypeScript parses their digits to the + # infinity of their sign, rounding to nearest, so they compare as that. + huge = 10**400 + # The least integer that rounds to infinity, and the one below it. + rounds_up, rounds_down = 2**1024 - 2**970, 2**1024 - 2**970 - 1 + for cached, source in [ + (huge, huge + 1), + (rounds_up, huge), + (-rounds_up, -huge), + (rounds_down, 1.7976931348623157e308), + ]: + assert diff_shadow_mismatch_json({"n": cached}, {"n": source}) == ([], 0) + assert diff_shadow_mismatch_json(nest_members(34, cached), nest_members(34, source)) == ([], 0) + for cached, source in [(huge, -huge), (1e308, huge), (rounds_down, huge)]: + actual, count = diff_shadow_mismatch_json({"n": cached}, {"n": source}) + assert_entries(actual, [{"path": "/n", "cached": cached, "source": source}]) + assert count == 1 + actual, count = diff_shadow_mismatch_json(nest_members(34, cached), nest_members(34, source)) + expected = {"path": "/n" * 32, "cached": nest_members(2, cached), "source": nest_members(2, source)} + assert_entries(actual, [expected]) + assert count == 1 + + # The renderings differ, so the warning walks them and finds no difference. + fields, failure = render_shadow_mismatch_log(VALUES_ONLY, {"n": huge}, {"n": huge + 1}) + assert failure is None + assert fields == {"mismatchPaths": [], "mismatchPathCount": 0, "diffJson": "[]"} + + +def test_compares_json_primitives_by_type_past_the_depth_cap(): + path = "/n" * 32 + entries, count = diff_shadow_mismatch_json(nest_members(34, True), nest_members(34, 1)) + assert_entries(entries, [{"path": path, "cached": nest_members(2, True), "source": nest_members(2, 1)}]) + assert count == 1 + assert diff_shadow_mismatch_json(nest_members(34, [1, None]), nest_members(34, [1.0, None])) == ([], 0) + precise, rounded = nest_members(34, 9007199254740993), nest_members(34, 9007199254740992) + assert diff_shadow_mismatch_json(precise, rounded) == ([], 0) + + +def test_walks_members_named_like_javascript_prototype_members(): + cached = json.loads('{"constructor":1,"__proto__":{"a":1}}') + source = json.loads('{"toString":"x","valueOf":null,"__proto__":{"a":2}}') + entries = [ + {"path": "/__proto__/a", "cached": 1, "source": 2}, + {"path": "/constructor", "cached": 1}, + {"path": "/toString", "source": "x"}, + {"path": "/valueOf", "source": None}, + ] + + actual, count = diff_shadow_mismatch_json(cached, source) + assert_entries(actual, entries) + assert count == 4 + fields, failure = render_shadow_mismatch_log(VALUES_ONLY, cached, source) + assert failure is None + assert fields["mismatchPaths"] == [entry["path"] for entry in entries] + assert_entries(json.loads(fields["diffJson"]), entries) + + +def test_orders_lone_surrogate_member_names_by_utf16_code_unit(): + # Cached JSON can carry lone surrogates, as ECMAScript strings do. By code + # unit they sort around the pair D83D DE00 of the emoji; code-point order + # would put U+DC00 and U+E000 ahead of it. + order = ["\ud800", "😀", "\udc00", ""] + cached = {name: 1 for name in reversed(order)} + source = {name: 2 for name in order} + entries = [{"path": f"/{name}", "cached": 1, "source": 2} for name in order] + + actual, count = diff_shadow_mismatch_json(cached, source) + assert_entries(actual, entries) + assert count == 4 + fields, failure = render_shadow_mismatch_log(VALUES_ONLY, cached, source) + assert failure is None + assert fields["mismatchPaths"] == [entry["path"] for entry in entries] + assert fields["mismatchPathCount"] == 4 + # Each lone unit is escaped, so the field is valid UTF-8 JSON. + assert fields["diffJson"] == ( + '[{"path":"/\\ud800","cached":1,"source":2},{"path":"/😀","cached":1,"source":2},' + '{"path":"/\\udc00","cached":1,"source":2},{"path":"/","cached":1,"source":2}]' + ) + + +def test_compares_own_members_in_a_depth_capped_pair(): + def nest(leaf): + return json.loads('{"a":' * 32 + leaf + "}" * 32) + + cached, source = nest('{"__proto__":{}}'), nest('{"b":{}}') + entries = [{"path": "/a" * 32, "cached": {"__proto__": {}}, "source": {"b": {}}}] + + actual, count = diff_shadow_mismatch_json(cached, source) + assert_entries(actual, entries) + assert count == 1 + fields, failure = render_shadow_mismatch_log(VALUES_ONLY, cached, source) + assert failure is None + assert fields["mismatchPaths"] == ["/a" * 32] + assert_entries(json.loads(fields["diffJson"]), entries) + + +def test_cuts_entries_at_64_in_walk_order_and_counts_every_difference(): + cached = {f"m{index:02d}": 0 for index in range(69, -1, -1)} + source = {f"m{index:02d}": 1 for index in range(69, -1, -1)} + entries = [{"path": f"/m{index:02d}", "cached": 0, "source": 1} for index in range(64)] + + actual, count = diff_shadow_mismatch_json(cached, source) + assert_entries(actual, entries) + assert count == 70 + assert render_shadow_mismatch_log(PATHS_ONLY, cached, source) == ( + {"mismatchPaths": [entry["path"] for entry in entries], "mismatchPathCount": 70}, + None, + ) + + +def test_clamps_each_pointer_to_512_bytes_without_splitting_utf8(): + cached = {"k" * 511: 1, "k" * 600: 1, "é" * 300: 1} + source = {"k" * 511: 2, "k" * 600: 2, "é" * 300: 2} + + actual, count = diff_shadow_mismatch_json(cached, source) + assert_entries( + actual, + [ + {"path": "/" + "k" * 511, "cached": 1, "source": 2}, + {"path": "/" + "k" * 497 + TRUNCATION_MARKER, "cached": 1, "source": 2}, + {"path": "/" + "é" * 248 + TRUNCATION_MARKER, "cached": 1, "source": 2}, + ], + ) + assert count == 3 + + +def test_never_builds_a_whole_pointer_under_a_deep_chain_of_long_names(): + # Copying the pointer prefix for each member would cost members times + # pointer length; the walk builds only the clamped pointer of a retained entry. + names = [f"{level:02d}" + "x" * 100_000 for level in range(31)] + cached = {f"m{index:03d}": 0 for index in range(100)} + source = {f"m{index:03d}": 1 for index in range(100)} + for name in reversed(names): + cached, source = {name: cached}, {name: source} + path = "/" + names[0][:497] + TRUNCATION_MARKER + + tracemalloc.start() + try: + actual, count = diff_shadow_mismatch_json(cached, source) + peak = tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + assert_entries(actual, [{"path": path, "cached": 0, "source": 1}] * 64) + assert count == 100 + # A whole pointer here is 3.1 MB, and the prefixes along the chain 50 MB. + assert peak < 1_000_000 + + +def test_replaces_each_side_over_1_kib_with_its_clamped_json_text(): + numbers = [1000] * 300 + cached = { + "accent": "é" * 600, + "blob": "x" * 2000, + "edge": "e" * 1022, + "extra": "z" * 1100, + "list": numbers, + } + source = {"accent": "a", "blob": "y", "edge": "f" * 1022, "list": [1000]} + + actual, count = diff_shadow_mismatch_json(cached, source) + assert_entries( + actual, + [ + {"path": "/accent", "cached": '"' + "é" * 504 + TRUNCATION_MARKER, "source": "a"}, + {"path": "/blob", "cached": '"' + "x" * 1009 + TRUNCATION_MARKER, "source": "y"}, + {"path": "/edge", "cached": "e" * 1022, "source": "f" * 1022}, + {"path": "/extra", "cached": '"' + "z" * 1009 + TRUNCATION_MARKER}, + { + "path": "/list", + "cached": json.dumps(numbers, separators=(",", ":"))[:1010] + TRUNCATION_MARKER, + "source": [1000], + }, + ], + ) + assert count == 5 + + +def test_caps_the_walk_at_32_reference_tokens_however_deep_the_difference_lies(): + actual, count = diff_shadow_mismatch_json(nest_members(40, 1), nest_members(40, 2)) + assert_entries(actual, [{"path": "/n" * 32, "cached": nest_members(8, 1), "source": nest_members(8, 2)}]) + assert count == 1 + clamped = '{"n":' * 202 + TRUNCATION_MARKER + actual, count = diff_shadow_mismatch_json(nest_members(300, 1), nest_members(300, 2)) + assert_entries(actual, [{"path": "/n" * 32, "cached": clamped, "source": clamped}]) + assert count == 1 + + +def test_compares_equal_containers_past_the_depth_cap_regardless_of_order_and_depth(): + cached, source = "leaf", "leaf" + for level in range(100_000): + cached = {"a": level, "next": cached} + source = {"next": source, "a": level} + + assert diff_shadow_mismatch_json(cached, source) == ([], 0) + + +@pytest.mark.parametrize( + ("plan", "fields"), + [ + (MismatchLogPlan(False, "off"), {}), + (PATHS_ONLY, {"mismatchPaths": ["/email"], "mismatchPathCount": 1}), + ( + VALUES_ONLY, + { + "mismatchPaths": ["/email"], + "mismatchPathCount": 1, + "diffJson": '[{"path":"/email","cached":"old@example.com","source":"new@example.com"}]', + }, + ), + (VALUE_ONLY, {"cachedValueJson": CACHED_USER_JSON, "sourceValueJson": SOURCE_USER_JSON}), + ( + MismatchLogPlan(True, "paths"), + { + "mismatchPaths": ["/email"], + "mismatchPathCount": 1, + "cachedValueJson": CACHED_USER_JSON, + "sourceValueJson": SOURCE_USER_JSON, + }, + ), + ( + EVERYTHING, + { + "mismatchPaths": ["/email"], + "mismatchPathCount": 1, + "diffJson": '[{"path":"/email","cached":"old@example.com","source":"new@example.com"}]', + "cachedValueJson": CACHED_USER_JSON, + "sourceValueJson": SOURCE_USER_JSON, + }, + ), + ], + ids=["neither", "paths", "values", "value", "value-paths", "value-values"], +) +def test_renders_exactly_the_requested_fields(plan, fields): + content = render_shadow_mismatch_log(plan, CACHED_USER, SOURCE_USER) + + assert content == (fields, None) + assert list(content[0]) == list(fields) + + +def test_renders_no_entries_when_the_renderings_hold_the_same_json(): + no_entries = {"mismatchPaths": [], "mismatchPathCount": 0, "diffJson": "[]"} + + # The default comparator tells a tuple from a list; both render as arrays. + assert render_shadow_mismatch_log(EVERYTHING, {"tags": ("a",)}, {"tags": ["a"]}) == ( + {**no_entries, "cachedValueJson": '{"tags":["a"]}', "sourceValueJson": '{"tags":["a"]}'}, + None, + ) + # Different text, same JSON values. + assert render_shadow_mismatch_log(VALUES_ONLY, {"n": 1.0, "m": 2}, {"m": 2, "n": 1}) == (no_entries, None) + + +@pytest.mark.parametrize( + "unrenderable", + [circular(), object(), {1, 2}, math.nan, math.inf, {"nested": UNDEFINED}, b"bytes"], + ids=["cycle", "object", "set", "nan", "inf", "nested-undefined", "bytes"], +) +def test_nulls_the_fields_fed_by_an_unrenderable_root_with_render_error(unrenderable): + assert render_shadow_mismatch_log(EVERYTHING, unrenderable, {"version": 2}) == ( + {**FAILED_DIFF, "cachedValueJson": None, "sourceValueJson": '{"version":2}'}, + "render_error", + ) + assert render_shadow_mismatch_log(EVERYTHING, {"version": 2}, unrenderable) == ( + {**FAILED_DIFF, "cachedValueJson": '{"version":2}', "sourceValueJson": None}, + "render_error", + ) + assert render_shadow_mismatch_log(PATHS_ONLY, {"version": 2}, unrenderable) == ( + {"mismatchPaths": None, "mismatchPathCount": None}, + "render_error", + ) + + +def test_reports_an_absent_root_as_one_side_at_the_root_pointer(): + assert render_shadow_mismatch_log(EVERYTHING, UNDEFINED, {"version": 2}) == ( + { + "mismatchPaths": [""], + "mismatchPathCount": 1, + "diffJson": '[{"path":"","source":{"version":2}}]', + "cachedValueJson": None, + "sourceValueJson": '{"version":2}', + }, + None, + ) + assert render_shadow_mismatch_log(EVERYTHING, {"version": 2}, UNDEFINED) == ( + { + "mismatchPaths": [""], + "mismatchPathCount": 1, + "diffJson": '[{"path":"","cached":{"version":2}}]', + "cachedValueJson": '{"version":2}', + "sourceValueJson": None, + }, + None, + ) + assert render_shadow_mismatch_log(PATHS_ONLY, UNDEFINED, None) == ( + {"mismatchPaths": [""], "mismatchPathCount": 1}, + None, + ) + # A stored None is a value, so it stays distinct from absence. + fields, _ = render_shadow_mismatch_log(VALUES_ONLY, UNDEFINED, None) + assert fields["diffJson"] == '[{"path":"","source":null}]' + # Absence beside a side with no rendering is still a render failure. + assert render_shadow_mismatch_log(EVERYTHING, UNDEFINED, object()) == ( + {**FAILED_DIFF, "cachedValueJson": None, "sourceValueJson": None}, + "render_error", + ) + # A custom comparator may call two absent values different; no field differs. + assert render_shadow_mismatch_log(EVERYTHING, UNDEFINED, UNDEFINED) == ( + { + "mismatchPaths": [], + "mismatchPathCount": 0, + "diffJson": "[]", + "cachedValueJson": None, + "sourceValueJson": None, + }, + None, + ) + + +def test_renders_each_side_once_for_every_field_it_feeds(): + cached, source = Stateful("a"), Stateful("b") + + assert render_shadow_mismatch_log(EVERYTHING, cached, source) == ( + { + "mismatchPaths": ["/email"], + "mismatchPathCount": 1, + "diffJson": '[{"path":"/email","cached":"a","source":"b"}]', + "cachedValueJson": '{"email":"a","render":1}', + "sourceValueJson": '{"email":"b","render":1}', + }, + None, + ) + assert (cached.renders, source.renders) == (1, 1) + + +def test_clamps_diff_json_to_8_kib_with_the_truncation_marker(): + cached = {f"m{index:02d}": "x" * 500 for index in range(20)} + source = {f"m{index:02d}": "y" * 500 for index in range(20)} + entries = [{"path": f"/m{index:02d}", "cached": "x" * 500, "source": "y" * 500} for index in range(20)] + # The 8 KiB bound is the record's, so it is written out rather than taken + # from the implementation's constant. + diff_json = ( + json.dumps(entries, separators=(",", ":"))[: 8192 - len(TRUNCATION_MARKER)] + TRUNCATION_MARKER + ) + + assert render_shadow_mismatch_log(VALUES_ONLY, cached, source) == ( + { + "mismatchPaths": [entry["path"] for entry in entries], + "mismatchPathCount": 20, + "diffJson": diff_json, + }, + None, + ) + assert len(diff_json.encode()) == 8192 + + +def test_clamps_each_built_in_diff_json_side_to_1_kib(): + fields, _ = render_shadow_mismatch_log(VALUES_ONLY, {"blob": "x" * 2000}, {"blob": "y"}) + + assert json.loads(fields["diffJson"]) == [ + {"path": "/blob", "cached": '"' + "x" * 1009 + TRUNCATION_MARKER, "source": "y"} + ] + + +def test_takes_the_entries_from_the_hook_and_never_runs_the_built_in_walk(monkeypatch): + cached, source = {"id": 1}, {"id": 2} + calls = [] + + def hook(*args): + calls.append(args) + return [ + {"path": "/profile/email", "cached": "old@example.com", "source": "new@example.com"}, + MappingProxyType({"path": "/roles"}), + ] + + def walk(cached, source): + raise AssertionError("the built-in walk ran") + + monkeypatch.setattr(_shadow_log, "_walk", walk) + assert render_shadow_mismatch_log(EVERYTHING, cached, source, hook) == ( + { + "mismatchPaths": ["/profile/email", "/roles"], + "mismatchPathCount": 2, + "diffJson": '[{"path":"/profile/email","cached":"old@example.com","source":"new@example.com"},' + '{"path":"/roles"}]', + "cachedValueJson": '{"id":1}', + "sourceValueJson": '{"id":2}', + }, + None, + ) + assert len(calls) == 1 + assert calls[0][0] is cached and calls[0][1] is source + + +def test_renders_neither_root_side_when_the_hook_supplies_entries_and_value_is_off(): + cached, source = Stateful("a"), Stateful("b") + + assert render_shadow_mismatch_log(VALUES_ONLY, cached, source, lambda cached, source: []) == ( + {"mismatchPaths": [], "mismatchPathCount": 0, "diffJson": "[]"}, + None, + ) + assert (cached.renders, source.renders) == (0, 0) + + +class FailingSide(dict): + def __getitem__(self, name): + if name == "cached": + raise RuntimeError("side unavailable") + return super().__getitem__(name) + + +def raises(cached, source): + raise RuntimeError("hook failed") + + +def cancels(cached, source): + raise asyncio.CancelledError() + + +async def coroutine_hook(cached, source): + return [{"path": "/email"}] + + +@pytest.mark.parametrize( + "hook", + [ + raises, + cancels, + coroutine_hook, + lambda cached, source: ({"path": "/email"},), + lambda cached, source: [{"cached": 1}], + lambda cached, source: [FailingSide(path="/email", cached=1)], + ], + ids=["raises", "raises-cancelled", "coroutine", "tuple", "member-without-path", "failing-side"], +) +def test_fails_closed_with_hook_error_and_no_fallback(hook): + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert render_shadow_mismatch_log(EVERYTHING, CACHED_USER, SOURCE_USER, hook) == ( + {**FAILED_DIFF, "cachedValueJson": CACHED_USER_JSON, "sourceValueJson": SOURCE_USER_JSON}, + "hook_error", + ) + assert render_shadow_mismatch_log(PATHS_ONLY, CACHED_USER, SOURCE_USER, hook) == ( + {"mismatchPaths": None, "mismatchPathCount": None}, + "hook_error", + ) + gc.collect() + + +def test_logs_the_hook_entries_as_returned_and_reads_only_the_kept_ones(): + def hook(cached, source): + return [{"path": 7}, *({"path": f"/{index}"} for index in range(63)), None] + + assert render_shadow_mismatch_log(PATHS_ONLY, CACHED_USER, SOURCE_USER, hook) == ( + {"mismatchPaths": ["7", *(f"/{index}" for index in range(63))], "mismatchPathCount": 65}, + None, + ) + + +def test_closes_a_coroutine_hook_result_without_running_it(): + started, returned = [], [] + + async def work(): + started.append(True) + return [] + + def hook(cached, source): + returned.append(work()) + return returned[-1] + + assert render_shadow_mismatch_log(PATHS_ONLY, CACHED_USER, SOURCE_USER, hook)[1] == "hook_error" + assert started == [] + assert returned[0].cr_frame is None + + +def test_consumes_a_failing_future_hook_result_without_owning_it(): + loop = asyncio.new_event_loop() + unretrieved = [] + loop.set_exception_handler(lambda loop, context: unretrieved.append(context)) + try: + returned = [loop.create_future()] + assert render_shadow_mismatch_log(PATHS_ONLY, CACHED_USER, SOURCE_USER, lambda c, s: returned[0])[ + 1 + ] == ("hook_error") + future = returned.pop() + assert not future.done() + future.set_exception(RuntimeError("hook task failed")) + loop.run_until_complete(asyncio.sleep(0)) + del future + gc.collect() + finally: + loop.close() + assert unretrieved == [] + + +@pytest.mark.parametrize( + "unrenderable", + [object(), circular(), math.nan, {1}], + ids=["object", "cycle", "nan", "set"], +) +def test_keeps_the_hook_paths_and_nulls_diff_json_when_a_hook_side_has_no_rendering(unrenderable): + def hook(cached, source): + return [{"path": "/a", "cached": 1}, {"path": "/b", "source": unrenderable}] + + assert render_shadow_mismatch_log(VALUES_ONLY, CACHED_USER, SOURCE_USER, hook) == ( + {"mismatchPaths": ["/a", "/b"], "mismatchPathCount": 2, "diffJson": None}, + "render_error", + ) + assert render_shadow_mismatch_log(PATHS_ONLY, CACHED_USER, SOURCE_USER, hook) == ( + {"mismatchPaths": ["/a", "/b"], "mismatchPathCount": 2}, + None, + ) + + +def test_stops_rendering_hook_sides_at_the_first_side_with_no_rendering(): + later = Stateful("later") + + def hook(cached, source): + return [{"path": "/a", "cached": object()}, {"path": "/b", "source": later}] + + assert render_shadow_mismatch_log(VALUES_ONLY, CACHED_USER, SOURCE_USER, hook) == ( + {"mismatchPaths": ["/a", "/b"], "mismatchPathCount": 2, "diffJson": None}, + "render_error", + ) + assert later.renders == 0 + + +def test_omits_absent_hook_sides_from_diff_json_and_renders_present_ones_as_json(): + def hook(cached, source): + return [ + {"path": "/a"}, + {"path": "/b", "cached": 1}, + {"path": "/c", "source": None}, + {"path": "/d", "cached": UNDEFINED, "source": "x"}, + {"path": "/t", "cached": ("a",), "source": {"ok": True}}, + ] + + fields, failure = render_shadow_mismatch_log(VALUES_ONLY, CACHED_USER, SOURCE_USER, hook) + assert failure is None + assert fields["diffJson"] == ( + '[{"path":"/a"},{"path":"/b","cached":1},{"path":"/c","source":null},{"path":"/d","source":"x"},' + '{"path":"/t","cached":["a"],"source":{"ok":true}}]' + ) + + +def test_clamps_hook_paths_and_sides_and_cuts_at_64_with_the_total_count(): + long_path = "/" + "k" * 600 + clamped_path = "/" + "k" * 497 + TRUNCATION_MARKER + + def hook(cached, source): + return [ + {"path": long_path, "cached": "x" * 2000, "source": {"ok": True}}, + *({"path": f"/{index + 1}"} for index in range(69)), + ] + + retained = [f"/{index + 1}" for index in range(63)] + fields, failure = render_shadow_mismatch_log(VALUES_ONLY, CACHED_USER, SOURCE_USER, hook) + + assert failure is None + assert fields["mismatchPaths"] == [clamped_path, *retained] + assert fields["mismatchPathCount"] == 70 + assert json.loads(fields["diffJson"]) == [ + {"path": clamped_path, "cached": '"' + "x" * 1009 + TRUNCATION_MARKER, "source": {"ok": True}}, + *({"path": path} for path in retained), + ] + + +def test_does_not_call_the_hook_when_diff_is_off(): + calls = [] + + def hook(cached, source): + calls.append(True) + return [{"path": "/email"}] + + assert render_shadow_mismatch_log(VALUE_ONLY, CACHED_USER, SOURCE_USER, hook) == ( + {"cachedValueJson": CACHED_USER_JSON, "sourceValueJson": SOURCE_USER_JSON}, + None, + ) + assert calls == [] + + +def test_keeps_the_hook_entries_when_a_root_side_has_no_rendering(): + assert render_shadow_mismatch_log(EVERYTHING, object(), SOURCE_USER, lambda c, s: [{"path": "/a"}]) == ( + { + "mismatchPaths": ["/a"], + "mismatchPathCount": 1, + "diffJson": '[{"path":"/a"}]', + "cachedValueJson": None, + "sourceValueJson": SOURCE_USER_JSON, + }, + "render_error", + ) + + +def test_reports_hook_error_ahead_of_render_error(): + assert render_shadow_mismatch_log(EVERYTHING, circular(), SOURCE_USER, raises) == ( + {**FAILED_DIFF, "cachedValueJson": None, "sourceValueJson": SOURCE_USER_JSON}, + "hook_error", + ) diff --git a/python/tests/test_shadow_mismatch_logging.py b/python/tests/test_shadow_mismatch_logging.py new file mode 100644 index 00000000..959c0ca7 --- /dev/null +++ b/python/tests/test_shadow_mismatch_logging.py @@ -0,0 +1,797 @@ +"""Confirmed-mismatch warnings: fields per leaf, the hook, failures and their outcome event.""" + +from __future__ import annotations + +import asyncio +import gc +import json +import weakref + +import pytest +from formal.executor import Executor + +from dialcache import UNDEFINED, ConfigError, DialCache, Policy, _shadow_log +from dialcache import cache as cache_module +from dialcache.protocol import Frame +from dialcache.serializer import JSON_UNDEFINED_SENTINEL + +MESSAGE = "DialCache shadow validation mismatch: %s" +CACHED = {"id": "123", "version": 1} +SOURCE = {"id": "123", "version": 2} +AGE_MS = 30_500 +PATH_FIELDS = {"mismatchPaths": ["/version"], "mismatchPathCount": 1} +DIFF_JSON = '[{"path":"/version","cached":1,"source":2}]' +VALUE_FIELDS = {"cachedValueJson": '{"id":"123","version":1}', "sourceValueJson": '{"id":"123","version":2}'} +EVERY_FIELD = {**PATH_FIELDS, "diffJson": DIFF_JSON, **VALUE_FIELDS} + + +@pytest.fixture +def executor(): + instance = Executor() + try: + yield instance + finally: + instance.close() + + +class Logger: + """Records each warning, then fails or returns as the test asks.""" + + def __init__(self, deliver=None): + self.calls = [] + self.deliver = deliver + + def warning(self, *args): + self.calls.append(args) + if self.deliver is not None: + return self.deliver() + return None + + def warnings(self): + return [details for message, details in self.calls if message == MESSAGE] + + +class Harness: + def __init__( + self, + executor, + *, + logger=None, + provider=None, + payload=None, + namespace="urn", + shadow_max_in_flight=1, + metrics=None, + ): + self.executor = executor + self.events = [] + self.logger = logger if logger is not None else Logger() + self.payload = json.dumps(CACHED, separators=(",", ":")) if payload is None else payload + harness = self + + class Redis: + def read(self, request, context): + return Frame(int(executor.clock.wall_ms()) - AGE_MS, harness.payload) + + def write(self, request): + return None + + self.cache = DialCache( + namespace=namespace, + redis=Redis(), + clock=executor.clock, + metrics=metrics or self.events.append, + logger=self.logger, + policy_provider=provider, + shadow_max_in_flight=shadow_max_in_flight, + ) + + def get(self, shadow, *, source=SOURCE, dark=False, key="123", use_case="Mismatch", **options): + policy = { + "ttl_sec": {"remote": 60}, + "ramp": {"remote": 0 if dark else 100}, + "shadow": {"ramp": 100, **shadow}, + } + + async def call(): + with self.cache.enable(): + return await self.cache.get_or_load( + lambda: source, + key=key, + key_type="user_id", + use_case=use_case, + default_config=policy, + **options, + ) + + return self.executor.finish(call()) + + def named(self, name): + return [event for event in self.events if event["event"] == name] + + def outcomes(self, name): + return [event["outcome"] for event in self.named(name)] + + def logging_events(self): + return self.named("shadowMismatchLogging") + + def config_errors(self): + return [event for event in self.named("error") if event["error"] == "config_resolution"] + + +def labels(use_case="Mismatch", namespace="urn"): + return {"cacheNamespace": namespace, "useCase": use_case, "keyType": "user_id"} + + +def logging_event(outcome, use_case="Mismatch", namespace="urn"): + return {"event": "shadowMismatchLogging", **labels(use_case, namespace), "outcome": outcome} + + +def warning(fields, *, use_case="Mismatch", namespace="urn", key="123"): + return { + **labels(use_case, namespace), + "outcome": "mismatch", + "cachedValueAgeSeconds": AGE_MS / 1000, + "cacheKey": f"{namespace}:user_id:{key}#{use_case}", + **fields, + } + + +@pytest.mark.parametrize("dark", [False, True], ids=["served-hit", "ramped-down"]) +def test_logs_every_field_in_order_for_the_shorthand_and_one_logged_event(executor, dark): + harness = Harness(executor) + + result = harness.get({"logMismatches": True}, dark=dark) + + assert result == (SOURCE if dark else CACHED) + assert harness.outcomes("shadowValidation") == ["mismatch"] + assert harness.logger.calls == [(MESSAGE, warning(EVERY_FIELD))] + assert list(harness.logger.calls[0][1]) == [ + "cacheNamespace", + "useCase", + "keyType", + "outcome", + "cachedValueAgeSeconds", + "cacheKey", + "mismatchPaths", + "mismatchPathCount", + "diffJson", + "cachedValueJson", + "sourceValueJson", + ] + assert harness.logging_events() == [logging_event("logged")] + + +@pytest.mark.parametrize( + ("shadow", "fields"), + [ + ({"mismatch_logging": {"diff": "paths"}}, PATH_FIELDS), + ({"mismatchLogging": {"diff": "values", "value": False}}, {**PATH_FIELDS, "diffJson": DIFF_JSON}), + ({"mismatch_logging": {"value": True}}, VALUE_FIELDS), + ({"mismatch_logging": {"value": True, "diff": "paths"}}, {**PATH_FIELDS, **VALUE_FIELDS}), + ], + ids=["paths", "values-without-value", "value", "value-with-paths"], +) +def test_logs_exactly_the_fields_each_leaf_selects(executor, shadow, fields): + harness = Harness(executor) + + harness.get(shadow) + + assert harness.logger.warnings() == [warning(fields)] + assert list(harness.logger.warnings()[0])[6:] == list(fields) + assert harness.logging_events() == [logging_event("logged")] + + +def test_logs_the_value_age_the_verdict_observed(executor): + harness = Harness(executor) + + harness.get({"mismatch_logging": {"diff": "paths"}}) + + assert [event["seconds"] for event in harness.named("shadowAge")] == [30.5] + assert harness.logger.warnings()[0]["cachedValueAgeSeconds"] == 30.5 + + +def test_emits_one_event_per_confirmed_mismatch_after_its_verdict_and_none_for_a_match(executor): + harness = Harness(executor) + events = harness.events + harness.logger.deliver = lambda: events.append({"event": "warning"}) + shadow = {"mismatch_logging": {"diff": "paths"}} + + harness.get(shadow, source=CACHED) + assert harness.outcomes("shadowValidation") == ["match"] + assert harness.logging_events() == [] + harness.get(shadow, key="b") + harness.get(shadow, key="c") + + assert harness.outcomes("shadowValidation") == ["match", "mismatch", "mismatch"] + assert harness.logging_events() == [logging_event("logged"), logging_event("logged")] + sequence = [ + event["event"] + for event in events + if event["event"] in ("shadowValidation", "shadowAge", "warning", "shadowMismatchLogging") + ] + assert sequence == [ + "shadowValidation", + "shadowAge", + *["shadowValidation", "shadowAge", "warning", "shadowMismatchLogging"] * 2, + ] + + +@pytest.mark.parametrize("shadow", [{}, {"log_mismatches": False}, {"mismatch_logging": {}}]) +def test_logs_nothing_and_emits_no_event_with_logging_off(executor, shadow): + harness = Harness(executor) + + harness.get(shadow) + + assert harness.outcomes("shadowValidation") == ["mismatch"] + assert harness.logger.calls == [] + assert harness.logging_events() == [] + + +def test_builds_labels_from_the_key_rather_than_from_mutated_metric_events(executor): + def mutate(event): + harness.events.append(dict(event)) + event["cacheNamespace"] = "mutated-by-metrics" + event["cacheKey"] = "injected-by-metrics" + + harness = Harness(executor, namespace="private-namespace", metrics=mutate) + + harness.get({"logMismatches": True}) + + assert harness.logger.warnings() == [warning(EVERY_FIELD, namespace="private-namespace")] + assert harness.logging_events() == [logging_event("logged", namespace="private-namespace")] + + +def test_logs_the_documented_get_user_example(executor): + cached = { + "id": "u_123", + "email": "ana@example.com", + "plan": "free", + "roles": ["viewer"], + "profile": {"name": "Ana", "locale": "en-US"}, + } + source = { + "id": "u_123", + "email": "ana@example.com", + "plan": "pro", + "roles": ["viewer", "editor"], + "profile": {"name": "Ana", "locale": "en-GB", "timezone": "Europe/London"}, + } + paths = { + "mismatchPaths": ["/plan", "/profile/locale", "/profile/timezone", "/roles"], + "mismatchPathCount": 4, + } + diff_json = ( + '[{"path":"/plan","cached":"free","source":"pro"},' + '{"path":"/profile/locale","cached":"en-US","source":"en-GB"},' + '{"path":"/profile/timezone","source":"Europe/London"},' + '{"path":"/roles","cached":["viewer"],"source":["viewer","editor"]}]' + ) + for diff, fields in [("paths", paths), ("values", {**paths, "diffJson": diff_json})]: + harness = Harness(Executor(), namespace="users-api", payload=json.dumps(cached)) + try: + harness.get({"mismatch_logging": {"diff": diff}}, source=source, key="u_123", use_case="GetUser") + assert harness.logger.warnings() == [ + warning(fields, use_case="GetUser", namespace="users-api", key="u_123") + ] + finally: + harness.executor.close() + + +def test_clamps_the_key_and_every_value_field(executor): + key = "k" * 2148 + cached = {"id": key, "content": "🙂" * 8192} + source = {"id": key, "content": "s" * 8193} + harness = Harness(executor, payload=json.dumps(cached)) + + harness.get({"log_mismatches": True}, source=source, key=key) + + (details,) = harness.logger.warnings() + assert "loggingOutcome" not in details + for field, limit in [("cacheKey", 2048), ("cachedValueJson", 8192), ("sourceValueJson", 8192)]: + assert len(details[field].encode()) <= limit + assert details[field].endswith("...[truncated]") + assert "�" not in details[field] + assert details["mismatchPaths"] == ["/content"] + assert details["mismatchPathCount"] == 1 + assert len(details["diffJson"].encode()) <= 8192 + (entry,) = json.loads(details["diffJson"]) + assert list(entry) == ["path", "cached", "source"] + for side in (entry["cached"], entry["source"]): + assert len(side.encode()) <= 1024 + assert side.startswith('"') and side.endswith("...[truncated]") + assert "�" not in side + assert harness.logging_events() == [logging_event("logged")] + + +@pytest.mark.parametrize("api", ["cached", "get_or_load"]) +def test_logs_hook_entries_in_place_of_the_built_in_walk(executor, api, monkeypatch): + harness = Harness(executor) + calls = [] + + def hook(cached, source): + calls.append((cached, source)) + return [ + {"path": "/custom", "cached": "c", "source": "s"}, + {"path": "/only-path"}, + {"path": "/one-side", "source": {"nested": True}}, + ] + + def walk(cached, source): + raise AssertionError("the built-in walk ran") + + monkeypatch.setattr(_shadow_log, "_walk", walk) + shadow = {"ttl_sec": {"remote": 60}, "shadow": {"ramp": 100, "log_mismatches": True}} + if api == "cached": + + @harness.cache.cached( + key_type="user_id", + cache_key=lambda: "123", + use_case="Mismatch", + default_config=shadow, + shadow_mismatch_diff=hook, + ) + async def load(): + return SOURCE + + async def call(): + with harness.cache.enable(): + return await load() + else: + + async def call(): + with harness.cache.enable(): + return await harness.cache.get_or_load( + lambda: SOURCE, + key="123", + key_type="user_id", + use_case="Mismatch", + default_config=shadow, + shadow_mismatch_diff=hook, + ) + + assert executor.finish(call()) == CACHED + assert calls == [(CACHED, SOURCE)] + assert harness.logger.warnings() == [ + warning( + { + "mismatchPaths": ["/custom", "/only-path", "/one-side"], + "mismatchPathCount": 3, + "diffJson": '[{"path":"/custom","cached":"c","source":"s"},{"path":"/only-path"},' + '{"path":"/one-side","source":{"nested":true}}]', + **VALUE_FIELDS, + } + ) + ] + assert harness.logging_events() == [logging_event("logged")] + + +def test_never_runs_the_hook_at_value_alone(executor): + harness = Harness(executor) + calls = [] + + harness.get( + {"mismatch_logging": {"value": True}}, shadow_mismatch_diff=lambda c, s: calls.append(1) or [] + ) + + assert harness.logger.warnings() == [warning(VALUE_FIELDS)] + assert calls == [] + + +async def async_hook(cached, source): + return [{"path": "/version"}] + + +def raising_hook(cached, source): + raise RuntimeError("diff hook unavailable") + + +@pytest.mark.parametrize("hook", [raising_hook, async_hook], ids=["raises", "coroutine"]) +def test_logs_null_diff_fields_and_hook_error_without_touching_the_verdict(executor, hook): + harness = Harness(executor) + + assert harness.get({"log_mismatches": True}, shadow_mismatch_diff=hook) == CACHED + + assert harness.outcomes("shadowValidation") == ["mismatch"] + (details,) = harness.logger.warnings() + assert details == warning( + { + "mismatchPaths": None, + "mismatchPathCount": None, + "diffJson": None, + **VALUE_FIELDS, + "loggingOutcome": "hook_error", + } + ) + assert list(details)[-1] == "loggingOutcome" + assert harness.logging_events() == [logging_event("hook_error")] + + +@pytest.mark.parametrize("comparator", [None, lambda cached, source: False], ids=["default", "custom"]) +def test_logs_render_error_for_a_source_value_without_a_json_rendering(executor, comparator): + harness = Harness(executor) + options = {} if comparator is None else {"shadow_comparator": comparator} + + harness.get({"log_mismatches": True}, source={"id": "123", "version": {1}}, dark=False, **options) + + assert harness.outcomes("shadowValidation") == ["mismatch"] + assert harness.logger.warnings() == [ + warning( + { + "mismatchPaths": None, + "mismatchPathCount": None, + "diffJson": None, + "cachedValueJson": '{"id":"123","version":1}', + "sourceValueJson": None, + "loggingOutcome": "render_error", + } + ) + ] + assert harness.logging_events() == [logging_event("render_error")] + + +def test_logs_a_cached_undefined_entry_as_an_absent_side(executor): + harness = Harness(executor, payload=JSON_UNDEFINED_SENTINEL) + + assert harness.get({"log_mismatches": True}, source={"id": "123"}) is UNDEFINED + + assert harness.outcomes("shadowValidation") == ["mismatch"] + assert harness.logger.warnings() == [ + warning( + { + "mismatchPaths": [""], + "mismatchPathCount": 1, + "diffJson": '[{"path":"","source":{"id":"123"}}]', + "cachedValueJson": None, + "sourceValueJson": '{"id":"123"}', + } + ) + ] + assert harness.logging_events() == [logging_event("logged")] + + +def test_passes_an_absent_cached_side_to_the_hook_as_the_raw_undefined(executor): + inputs = [] + + def hook(cached, source): + inputs.append((cached, source)) + return [{"path": "", "source": source}] + + harness = Harness(executor, payload=JSON_UNDEFINED_SENTINEL) + + result = harness.get( + {"mismatch_logging": {"diff": "values"}}, source={"id": "123"}, shadow_mismatch_diff=hook + ) + + assert result is UNDEFINED + # D13: the hook receives the raw values, so an absent side stays UNDEFINED + # rather than becoming None, which is JSON null. + ((cached, source),) = inputs + assert cached is UNDEFINED + assert source == {"id": "123"} + assert harness.logger.warnings() == [ + warning( + { + "mismatchPaths": [""], + "mismatchPathCount": 1, + "diffJson": '[{"path":"","source":{"id":"123"}}]', + } + ) + ] + assert harness.logging_events() == [logging_event("logged")] + + +def test_logs_hook_paths_without_rendering_values_at_diff_paths(executor): + harness = Harness(executor) + + harness.get( + {"mismatch_logging": {"diff": "paths"}}, + source={"id": "123", "version": object()}, + shadow_mismatch_diff=lambda cached, source: [{"path": "/version"}], + ) + + assert harness.logger.warnings() == [warning(PATH_FIELDS)] + assert harness.logging_events() == [logging_event("logged")] + + +def test_falls_back_to_a_metadata_warning_when_construction_raises(executor, monkeypatch): + def fail(*args): + raise RuntimeError("preview unavailable") + + monkeypatch.setattr(cache_module, "render_shadow_mismatch_log", fail) + harness = Harness(executor) + + harness.get({"log_mismatches": True}) + + assert harness.outcomes("shadowValidation") == ["mismatch"] + (details,) = harness.logger.warnings() + assert details == warning({"loggingOutcome": "render_error"}) + assert harness.logging_events() == [logging_event("render_error")] + + +def logger_raises(): + raise RuntimeError("logger unavailable") + + +def logger_cancels(): + raise asyncio.CancelledError() + + +async def logger_coroutine_fails(): + raise RuntimeError("logger transport unavailable") + + +async def logger_coroutine_delivers(): + return None + + +class Uninspectable: + """A logger result that raises when the awaitable check reads its class.""" + + @property + def __class__(self): + raise RuntimeError("logger result unavailable") + + +@pytest.mark.parametrize( + ("deliver", "outcome"), + [ + (logger_raises, "logger_error"), + (logger_cancels, "logger_error"), + (logger_coroutine_fails, "logger_error"), + (Uninspectable, "logger_error"), + (logger_coroutine_delivers, "logged"), + (lambda: "ignored", "logged"), + ], + ids=[ + "raises", + "raises-cancelled", + "coroutine-fails", + "returns-an-uninspectable-value", + "coroutine-delivers", + "returns-a-value", + ], +) +def test_records_the_logger_outcome_and_leaves_the_caller_unaffected(executor, deliver, outcome): + harness = Harness(executor, logger=Logger(deliver)) + + assert harness.get({"log_mismatches": True}) == CACHED + + assert harness.outcomes("shadowValidation") == ["mismatch"] + assert harness.logger.calls == [(MESSAGE, warning(EVERY_FIELD))] + assert harness.logging_events() == [logging_event(outcome)] + + +@pytest.mark.parametrize( + ("settle", "outcome"), + [ + (lambda future: future.set_result(None), "logged"), + (lambda future: future.set_exception(RuntimeError("logger transport unavailable")), "logger_error"), + (lambda future: future.cancel(), "logger_error"), + ], + ids=["resolves", "rejects", "cancelled"], +) +def test_waits_for_a_returned_future_before_recording_the_outcome(executor, settle, outcome): + delivery = executor.future() + harness = Harness(executor, logger=Logger(lambda: delivery)) + + harness.get({"log_mismatches": True}) + assert harness.outcomes("shadowValidation") == ["mismatch"] + assert len(harness.logger.warnings()) == 1 + assert harness.logging_events() == [] + + settle(delivery) + executor.drain() + assert harness.logging_events() == [logging_event(outcome)] + + +def test_records_no_outcome_while_a_returned_future_never_settles(executor): + harness = Harness(executor, logger=Logger(executor.future)) + + harness.get({"log_mismatches": True}) + for _ in range(5): + executor.drain() + + assert harness.outcomes("shadowValidation") == ["mismatch"] + assert len(harness.logger.warnings()) == 1 + assert harness.logging_events() == [] + + +class Watched(dict): + """A compared value the test can watch for collection.""" + + +def test_holds_no_compared_value_while_a_returned_future_never_settles(executor): + harness = Harness(executor, logger=Logger(executor.future)) + source = Watched(SOURCE) + watched = weakref.ref(source) + + assert harness.get({"log_mismatches": True}, source=source) == CACHED + del source + gc.collect() + + assert len(harness.logger.warnings()) == 1 + assert harness.logging_events() == [] + assert watched() is None + + +def test_frees_the_capacity_slot_before_a_returned_future_settles(executor): + harness = Harness(executor, logger=Logger(executor.future), shadow_max_in_flight=1) + + harness.get({"log_mismatches": True}, key="first") + harness.get({"log_mismatches": True}, key="second") + + assert harness.outcomes("shadowValidation") == ["mismatch", "mismatch"] + assert [details["cacheKey"] for details in harness.logger.warnings()] == [ + warning({}, key="first")["cacheKey"], + warning({}, key="second")["cacheKey"], + ] + assert harness.logging_events() == [] + + +def hook_unrenderable_side(cached, source): + return [{"path": "/version", "cached": object()}] + + +@pytest.mark.parametrize( + ("hook", "fields", "outcome"), + [ + (raising_hook, {"mismatchPaths": None, "mismatchPathCount": None, "diffJson": None}, "hook_error"), + (hook_unrenderable_side, {**PATH_FIELDS, "diffJson": None}, "render_error"), + ], + ids=["hook-raises", "hook-side-unrenderable"], +) +@pytest.mark.parametrize("logger", ["raises", "coroutine-fails", "pending"]) +def test_records_the_earlier_failure_rather_than_logger_error(executor, hook, fields, outcome, logger): + deliver = {"raises": logger_raises, "coroutine-fails": logger_coroutine_fails, "pending": executor.future} + harness = Harness(executor, logger=Logger(deliver[logger])) + + assert harness.get({"mismatch_logging": {"diff": "values"}}, shadow_mismatch_diff=hook) == CACHED + + assert harness.logger.calls == [(MESSAGE, warning({**fields, "loggingOutcome": outcome}))] + # Recorded at once, whatever the logger does, even while it is pending. + assert harness.logging_events() == [logging_event(outcome)] + + +@pytest.mark.parametrize("bad", ["paths", {"path": "/version"}, 1]) +def test_rejects_a_non_callable_shadow_mismatch_diff_where_each_api_captures_it(executor, bad): + cache = DialCache() + + async def load(): + return SOURCE + + with pytest.raises(ConfigError, match="shadow_mismatch_diff must be callable"): + cache.cached( + key_type="user_id", cache_key=lambda: "123", use_case="Invalid", shadow_mismatch_diff=bad + )(load) + # The rejected registration leaves the use case free for a valid one. + cache.cached(key_type="user_id", cache_key=lambda: "123", use_case="Invalid", shadow_mismatch_diff=None)( + load + ) + + async def call(): + return await cache.get_or_load( + lambda: SOURCE, key="123", key_type="user_id", use_case="InvalidInline", shadow_mismatch_diff=bad + ) + + with pytest.raises(ConfigError, match="shadow_mismatch_diff must be callable"): + executor.finish(call()) + + +def test_records_no_warning_and_one_config_resolution_for_an_invalid_runtime_shorthand(executor): + harness = Harness(executor, provider=lambda key: {"shadow": {"logMismatches": "yes"}}) + + harness.get({"log_mismatches": True}) + + assert harness.outcomes("shadowValidation") == ["mismatch"] + assert harness.config_errors() == [ + {"event": "error", **labels(), "layer": "remote", "error": "config_resolution", "inFallback": False} + ] + assert harness.logger.calls == [] + assert harness.logging_events() == [] + + +@pytest.mark.parametrize( + ("default", "runtime", "fields"), + [ + ({"mismatch_logging": {"value": True}}, {"mismatchLogging": {"diff": "everything"}}, VALUE_FIELDS), + ( + {"log_mismatches": True}, + {"logMismatches": "yes", "mismatchLogging": {"value": False, "diff": "paths"}}, + PATH_FIELDS, + ), + ( + {"log_mismatches": True}, + {"logMismatches": None, "mismatchLogging": {"value": "yes", "diff": "paths"}}, + PATH_FIELDS, + ), + ], + ids=["invalid-leaf", "malformed-shorthand", "every-input-invalid-but-one"], +) +def test_records_one_config_resolution_per_admitted_job_and_logs_the_valid_leaves( + executor, default, runtime, fields +): + harness = Harness(executor, provider=lambda key: {"shadow": runtime}) + + harness.get(default) + + assert len(harness.config_errors()) == 1 + assert harness.logger.warnings() == [warning(fields)] + assert harness.logging_events() == [logging_event("logged")] + + +def test_records_no_config_resolution_for_a_job_dropped_before_admission(executor): + gate = executor.future() + harness = Harness(executor, provider=lambda key: {"shadow": {"logMismatches": "yes"}}) + policy = {"ttl_sec": {"remote": 60}, "ramp": {"remote": 0}, "shadow": {"ramp": 100}} + + async def call(key, load): + with harness.cache.enable(): + return await harness.cache.get_or_load( + load, key=key, key_type="user_id", use_case="Mismatch", default_config=policy + ) + + held = executor.task(call("held", lambda: gate)) + executor.drain() + assert executor.finish(call("dropped", lambda: SOURCE)) == SOURCE + assert harness.outcomes("shadowValidation") == ["dropped"] + assert [event["useCase"] for event in harness.config_errors()] == ["Mismatch"] + gate.set_result(SOURCE) + executor.drain() + assert held.result() == SOURCE + assert len(harness.config_errors()) == 1 + + +@pytest.mark.parametrize( + ("default", "runtime", "fields"), + [ + ( + {"log_mismatches": True, "mismatch_logging": {"value": False}}, + None, + {**PATH_FIELDS, "diffJson": DIFF_JSON}, + ), + ({"log_mismatches": False, "mismatch_logging": {"diff": "paths"}}, None, PATH_FIELDS), + ({"mismatch_logging": {"diff": "paths"}}, {"log_mismatches": False}, None), + ({"mismatch_logging": {"value": False}}, {"logMismatches": True}, EVERY_FIELD), + ], + ids=[ + "true-beside-value-false", + "false-beside-paths", + "runtime-false-over-paths", + "runtime-true-over-value", + ], +) +def test_logs_the_recorded_shorthand_table(executor, default, runtime, fields): + provider = None if runtime is None else (lambda key: {"shadow": runtime}) + harness = Harness(executor, provider=provider) + + harness.get(default) + + assert harness.outcomes("shadowValidation") == ["mismatch"] + assert harness.config_errors() == [] + if fields is None: + assert harness.logger.calls == [] + assert harness.logging_events() == [] + else: + assert harness.logger.warnings() == [warning(fields)] + assert harness.logging_events() == [logging_event("logged")] + + +def test_disabled_overrides_every_inherited_mismatch_logging_leaf(executor): + default = {"log_mismatches": True, "mismatch_logging": {"value": True, "diff": "values"}} + disabled = Policy.disabled() + assert disabled.shadow == {"ramp": 0, "mismatch_logging": {"value": False, "diff": "off"}} + + killed = Harness(executor, provider=lambda key: Policy.disabled()) + assert killed.get(default) == SOURCE + assert killed.outcomes("shadowValidation") == [] + assert killed.logger.calls == [] + + # With only the shadow ramp restored, the verdict still runs, but the + # overlay's leaves replace both inherited ones. + restored = Harness(executor, provider=lambda key: {"shadow": {**disabled.shadow, "ramp": 100}}) + restored.get(default) + assert restored.outcomes("shadowValidation") == ["mismatch"] + assert restored.logger.calls == [] + assert restored.logging_events() == [] + assert restored.config_errors() == [] diff --git a/rust/README.md b/rust/README.md index 9f92184a..47a8e894 100644 --- a/rust/README.md +++ b/rust/README.md @@ -198,7 +198,13 @@ payloads remain inline; synchronous protocol helpers remain synchronous. The default CPU executor is shared across cache instances: two worker threads and two queued jobs. Admission never waits for queue space. Saturation or worker creation failure fails open: a read falls through to the source, and a failed -compression skips the write while preserving the source result. Custom runtimes +compression skips the write while preserving the source result. Shadow mismatch +warnings share this executor with caller reads: rendering a warning's content +and running its `mismatch_diff` hook queue beside the zstd decompression of +cache hits, so a burst of warnings can reject a compressed hit and send that +read to the source. Each instance runs at most `shadow_max_in_flight` shadow +jobs (default 1), so that setting and the number of instances in the process +bound how many warnings contend. Custom runtimes may supply their own bounded CPU executor. `StepRuntime` queues these jobs on its controlled executor for deterministic tests. A shadow job retains its capacity until its admitted CPU work finishes or is discarded, including after a deadline @@ -222,18 +228,30 @@ Default stale-recovery decode warnings omit error text that could contain cached values; JSON errors retain their category and line/column location. A custom `Logger` can inspect the original error when application-controlled details are needed. -Mismatch logging is opt-in, confirmed, bounded, and previews values through -the operation's `preview` (JSON for serde values). Default JSON previews retain -only an 8 KiB prefix while checking the entire serialization for errors. Preview -callbacks run through the bounded CPU executor after confirmation; they may run -on a worker thread. Queue rejection omits value previews but still logs the -confirmed mismatch. Diagnostic work holds shadow capacity until it finishes, -including after runtime shutdown. Observer and logger failures never change a -cache, source or maintenance result. +Mismatch logging is opt-in, confirmed and bounded. `ShadowPolicy::mismatch_logging` +chooses the warning's content: `value` logs the JSON of both values, and +`diff` at `MismatchDiff::Paths` logs RFC 6901 pointers to the differing +fields, which `MismatchDiff::Values` extends with what each side holds there; +`log_mismatches: Some(true)` is shorthand for both. Values are compared with +`PartialEq` or the operation's comparator; only the logged fields derive from +the values' `serde` JSON, which `Operation::new` and `register` render. A +`with_codec` or `register_custom` operation has none, so it logs value fields +through its `preview` and differences through its `mismatch_diff` hook. A +`preview` you set replaces the JSON for the whole warning, so the differences +of an operation with one also come only from the hook; setting `preview` to +`None` logs no values and leaves the differences to the hook as well. The +differences are computed over each value's whole rendering, so their cost +grows with the value's size. Rendering and hooks run through the bounded CPU executor after +confirmation; they may run on a worker thread. A failure leaves the field +unavailable and names its cause in the warning, and every warning attempt +emits one `ShadowMismatchLogging` event with its outcome. Queue rejection +still logs the metadata warning. Diagnostic work holds shadow capacity until +it finishes, including after runtime shutdown. Observer and logger failures +never change a cache, source or maintenance result. `MetricKind` maps every event to the metric names, labels and values shared with the TypeScript and Go exporters. `PrometheusObserver` (feature -`prometheus`) registers the nineteen collectors on a `prometheus::Registry` +`prometheus`) registers the twenty collectors on a `prometheus::Registry` under an optional prefix; clone one observer for every instance that exports to the same registry, because the `prometheus` crate cannot hand back an existing collector and a second registration of the same names is a diff --git a/rust/src/cpu_tests.rs b/rust/src/cpu_tests.rs index ed24409a..9b98b2e5 100644 --- a/rust/src/cpu_tests.rs +++ b/rust/src/cpu_tests.rs @@ -7,7 +7,7 @@ use std::time::Duration; use futures::future::BoxFuture; use parking_lot::Mutex; -use crate::observe::ShadowOutcome; +use crate::observe::{ShadowMismatchLoggingOutcome, ShadowOutcome}; use crate::protocol::{compress_payload, CompressionConfig}; use crate::testing::{StepRuntime, TestExecutor, WALL_EPOCH_MS}; use crate::*; @@ -104,6 +104,7 @@ fn operation(decodes: &Arc, shadow: bool) -> Operation { policy = policy.remote_ramp(0.0).shadow(ShadowPolicy { ramp: Some(100.0), log_mismatches: None, + mismatch_logging: None, }); } Operation::with_codec( @@ -335,8 +336,12 @@ async fn large_codec_phases_allow_an_independent_short_timer_to_progress() { } } +/// Warnings, and the logging outcomes the observer received. #[derive(Default)] -struct Logs(Mutex>); +struct Logs( + Mutex>, + Mutex>, +); impl Logger for Logs { fn log(&self, event: &LogEvent) { if let LogEvent::ShadowMismatch(details) = event { @@ -344,6 +349,19 @@ impl Logger for Logs { } } } +/// Forwards outcomes to the event recorder and logging outcomes to the logs. +struct Both(Arc, Arc); +impl Observer for Both { + fn observes_shadow_outcomes(&self) -> bool { + true + } + fn observe(&self, event: &Event) { + self.0.observe(event); + if let Event::ShadowMismatchLogging { outcome, .. } = event { + self.1 .1.lock().push(*outcome); + } + } +} fn preview_setup() -> ( TestExecutor, DialCache, @@ -362,7 +380,7 @@ fn preview_setup() -> ( let cache = DialCache::builder() .clock_arc(executor.clock.clone()) .runtime_arc(runtime.clone()) - .observer_arc(events.clone()) + .observer(Both(events.clone(), logs.clone())) .logger_arc(logs.clone()) .remote(SnapshotRemote { result: ReadResult::Hit(Frame { @@ -387,6 +405,7 @@ fn preview_call( let policy = Policy::default().remote_ttl_sec(60).shadow(ShadowPolicy { ramp: Some(100.0), log_mismatches: Some(true), + mismatch_logging: None, }); let result = if registered { let use_case = cache @@ -434,11 +453,17 @@ fn both_default_apis_prepare_bounded_previews_without_holding_the_caller() { assert!(cache.core.state.lock().shadows.is_empty()); let messages = logs.0.lock(); assert_eq!(messages.len(), 1); + let values = messages[0].values.as_ref().unwrap(); assert_eq!( - messages[0].cached_value_json, + values.cached_value_json, crate::preview::json_preview(&"é".repeat(10_000)) ); - assert_eq!(messages[0].source_value_json.as_deref(), Some("\"source\"")); + assert_eq!(values.source_value_json.as_deref(), Some("\"source\"")); + // The shorthand also logs the difference, here between the roots. + let diff = messages[0].diff.as_ref().unwrap(); + assert_eq!(diff.mismatch_paths, Some(vec![String::new()])); + assert_eq!(diff.mismatch_path_count, Some(1)); + assert_eq!(messages[0].logging_outcome, None); assert_eq!( *events.0.lock(), vec![ShadowOutcome::Mismatch, ShadowOutcome::Dropped] @@ -456,8 +481,17 @@ fn rejected_preview_cpu_work_still_logs_the_confirmed_mismatch() { assert!(cache.core.state.lock().shadows.is_empty()); let messages = logs.0.lock(); assert_eq!(messages.len(), 1); - assert!(messages[0].cached_value_json.is_none()); - assert!(messages[0].source_value_json.is_none()); + // The metadata warning remains, and its outcome says why it has no fields. + assert_eq!(messages[0].diff, None); + assert_eq!(messages[0].values, None); + assert_eq!( + messages[0].logging_outcome, + Some(ShadowMismatchLoggingOutcome::RenderError) + ); + assert_eq!( + *logs.1.lock(), + vec![ShadowMismatchLoggingOutcome::RenderError] + ); } #[test] @@ -482,11 +516,16 @@ fn custom_preview_failures_are_isolated_per_value_and_success_is_clamped() { executor.drain(); let messages = logs.0.lock(); assert_eq!(messages.len(), 1); - assert!(messages[0].cached_value_json.is_none()); + let values = messages[0].values.as_ref().unwrap(); + assert!(values.cached_value_json.is_none()); assert_eq!( - messages[0].source_value_json, + values.source_value_json, Some(crate::preview::preview_value(&"🙂".repeat(10_000))) ); + assert_eq!( + messages[0].logging_outcome, + Some(ShadowMismatchLoggingOutcome::RenderError) + ); assert!(cache.core.state.lock().shadows.is_empty()); } } diff --git a/rust/src/datadog.rs b/rust/src/datadog.rs index 60f49044..a0dbc7b6 100644 --- a/rust/src/datadog.rs +++ b/rust/src/datadog.rs @@ -101,6 +101,7 @@ pub fn metric_suffix(kind: MetricKind) -> &'static str { MetricKind::Coalesced => "coalesced.count", MetricKind::ShadowValidation => "shadow.count", MetricKind::ShadowValueAge => "shadow.value_age", + MetricKind::ShadowMismatchLogging => "shadow_mismatch_logging.count", MetricKind::FutureTimestampOffset => "future_timestamp_offset", MetricKind::StaleRecovery => "stale_recovery.count", MetricKind::StaleRecoveryValueAge => "stale_recovery.value_age", diff --git a/rust/src/engine.rs b/rust/src/engine.rs index c7b6dd43..80c0475a 100644 --- a/rust/src/engine.rs +++ b/rust/src/engine.rs @@ -67,8 +67,10 @@ impl Core { } } - pub(crate) fn log(&self, event: LogEvent) { - let _ = catch_unwind(AssertUnwindSafe(|| self.logger.log(&event))); + /// Deliver a log event; `false` when the logger panicked. Logger + /// failures never change results. + pub(crate) fn log(&self, event: LogEvent) -> bool { + catch_unwind(AssertUnwindSafe(|| self.logger.log(&event))).is_ok() } pub(crate) fn shadow_hook_enabled(&self) -> bool { diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 2a4d8ed1..5ae074d4 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -39,6 +39,9 @@ pub mod remote; pub mod runtime; mod scope; mod shadow; +mod shadow_log; +#[cfg(test)] +mod shadow_log_tests; #[cfg(feature = "test-util")] pub mod testing; mod use_case; @@ -59,10 +62,17 @@ pub use error::{BoxError, ConfigError, Error, FallbackTimeout, RemoteReadTimeout pub use identity::{normalize_args, ArgValue, Identity, IdentityError, IntoKeyId, Keys}; pub use local::{LocalEntry, LocalRead, LocalStore, LruLocalStore, StoredValue}; pub use metrics::MetricKind; -pub use observe::{Event, Labels, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails}; -pub use operation::{Comparator, Operation, Preview, RecoveryPredicate, SourceBudget}; +pub use observe::{ + Event, Labels, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails, + ShadowMismatchDiffDetails, ShadowMismatchValueDetails, +}; +pub use operation::{ + Comparator, MismatchDiffHook, MismatchEntry, Operation, Preview, RecoveryPredicate, + SourceBudget, +}; pub use policy::{ - Policy, PolicyDefaults, PolicyError, ResolvedPolicy, RuntimePolicy, ShadowPolicy, + MismatchDiff, MismatchLogging, Policy, PolicyDefaults, PolicyError, ResolvedPolicy, + RuntimePolicy, ShadowPolicy, }; #[cfg(feature = "redis")] pub use redis::{ diff --git a/rust/src/metrics.rs b/rust/src/metrics.rs index 1b1e8688..9a50817d 100644 --- a/rust/src/metrics.rs +++ b/rust/src/metrics.rs @@ -29,6 +29,8 @@ pub enum MetricKind { ShadowValidation, /// Seconds: validated value age at a match or mismatch verdict ([`Event::ShadowValueAge`]). ShadowValueAge, + /// Counter of confirmed-mismatch warning attempts ([`Event::ShadowMismatchLogging`]). + ShadowMismatchLogging, /// Seconds: how far a frame is dated ahead of the observing clock /// ([`Event::FutureTimestampOffset`]). FutureTimestampOffset, @@ -62,7 +64,7 @@ const LABEL_KEY_TYPE: &str = "key_type"; impl MetricKind { /// Every kind, in declaration order. - pub const ALL: [MetricKind; 19] = [ + pub const ALL: [MetricKind; 20] = [ MetricKind::Request, MetricKind::Miss, MetricKind::Disabled, @@ -71,6 +73,7 @@ impl MetricKind { MetricKind::Coalesced, MetricKind::ShadowValidation, MetricKind::ShadowValueAge, + MetricKind::ShadowMismatchLogging, MetricKind::FutureTimestampOffset, MetricKind::StaleRecovery, MetricKind::StaleRecoveryValueAge, @@ -95,6 +98,7 @@ impl MetricKind { Event::Coalesced { .. } => MetricKind::Coalesced, Event::ShadowValidation { .. } => MetricKind::ShadowValidation, Event::ShadowValueAge { .. } => MetricKind::ShadowValueAge, + Event::ShadowMismatchLogging { .. } => MetricKind::ShadowMismatchLogging, Event::FutureTimestampOffset { .. } => MetricKind::FutureTimestampOffset, Event::StaleRecovery { .. } => MetricKind::StaleRecovery, Event::StaleRecoveryValueAge { .. } => MetricKind::StaleRecoveryValueAge, @@ -120,6 +124,7 @@ impl MetricKind { MetricKind::Coalesced => "coalesced", MetricKind::ShadowValidation => "shadowValidation", MetricKind::ShadowValueAge => "shadowValueAge", + MetricKind::ShadowMismatchLogging => "shadowMismatchLogging", MetricKind::FutureTimestampOffset => "futureTimestampOffset", MetricKind::StaleRecovery => "staleRecovery", MetricKind::StaleRecoveryValueAge => "staleRecoveryValueAge", @@ -146,6 +151,7 @@ impl MetricKind { | MetricKind::Invalidation | MetricKind::Coalesced | MetricKind::ShadowValidation + | MetricKind::ShadowMismatchLogging | MetricKind::StaleRecovery | MetricKind::Compression ) @@ -203,6 +209,7 @@ impl MetricKind { ], MetricKind::ShadowValidation | MetricKind::ShadowValueAge + | MetricKind::ShadowMismatchLogging | MetricKind::StaleRecovery | MetricKind::StaleRecoveryValueAge => OUTCOME, MetricKind::Compression => &[ @@ -256,6 +263,13 @@ impl MetricKind { push_outcome_base(&mut labels, outcome); labels.push(("outcome", verdict.as_str().to_string())); } + Event::ShadowMismatchLogging { + labels: outcome, + outcome: logging, + } => { + push_outcome_base(&mut labels, outcome); + labels.push(("outcome", logging.as_str().to_string())); + } Event::StaleRecovery { labels: outcome, outcome: verdict, @@ -347,6 +361,7 @@ impl MetricKind { | Event::Invalidation { .. } | Event::Coalesced { .. } | Event::ShadowValidation { .. } + | Event::ShadowMismatchLogging { .. } | Event::StaleRecovery { .. } | Event::Compression { .. } => 1.0, } @@ -375,7 +390,7 @@ mod tests { use crate::observe::{ CoalescingScope, CompressionOperation, CompressionOutcome, DisabledReason, ErrorKind, Labels, Layer, MissReason, OutcomeLabels, RecoveryOutcome, SerializationOperation, - ShadowOutcome, + ShadowMismatchLoggingOutcome, ShadowOutcome, }; use std::sync::Arc; @@ -431,6 +446,10 @@ mod tests { outcome: ShadowOutcome::Mismatch, seconds: 0.25, }, + MetricKind::ShadowMismatchLogging => Event::ShadowMismatchLogging { + labels: outcome(), + outcome: ShadowMismatchLoggingOutcome::HookError, + }, MetricKind::FutureTimestampOffset => Event::FutureTimestampOffset { labels: base(), seconds: 0.25, @@ -489,7 +508,7 @@ mod tests { } let names: std::collections::HashSet<&str> = MetricKind::ALL.iter().map(|k| k.as_str()).collect(); - assert_eq!(names.len(), 19); + assert_eq!(names.len(), 20); } #[test] @@ -552,6 +571,15 @@ mod tests { ("outcome", "mismatch"), ]) ); + assert_eq!( + pairs(MetricKind::ShadowMismatchLogging), + owned(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("outcome", "hook_error"), + ]) + ); assert_eq!( pairs(MetricKind::Miss), owned(&[ @@ -615,6 +643,7 @@ mod tests { "invalidation", "coalesced", "shadowValidation", + "shadowMismatchLogging", "staleRecovery", "compression", ] diff --git a/rust/src/observe.rs b/rust/src/observe.rs index 2d38e062..f2be65f5 100644 --- a/rust/src/observe.rs +++ b/rust/src/observe.rs @@ -148,6 +148,26 @@ str_enum! { } } +str_enum! { + /// Outcomes of one confirmed-mismatch warning attempt. When more than one + /// failure occurs, `hook_error` ranks above `render_error`, which ranks + /// above `logger_error`. + ShadowMismatchLoggingOutcome { + /// Every requested field was built and the logger returned. + Logged => "logged", + /// The operation's `mismatch_diff` hook or the built-in walk panicked + /// or failed; `mismatch_paths`, `mismatch_path_count` and `diff_json` + /// are unavailable. + HookError => "hook_error", + /// A compared value had no JSON rendering, the operation has no + /// renderer for a requested field, or the bounded CPU executor + /// rejected the warning's work; the fields it feeds are unavailable. + RenderError => "render_error", + /// The logger panicked while receiving the warning. + LoggerError => "logger_error", + } +} + str_enum! { /// Terminal outcomes of stale-on-error recovery. RecoveryOutcome { @@ -305,6 +325,14 @@ pub enum Event { /// Observing wall clock minus the frame's `created_at_ms`, never negative. seconds: f64, }, + /// One confirmed-mismatch warning attempt, emitted after the warning and + /// only while mismatch logging is on. + ShadowMismatchLogging { + /// The identity the job validated. + labels: OutcomeLabels, + /// Whether the warning was built and delivered, or why not. + outcome: ShadowMismatchLoggingOutcome, + }, /// Positive offset in seconds of a frame dated after the observing clock. FutureTimestampOffset { /// The layer whose read decoded the frame. @@ -422,7 +450,11 @@ pub trait Observer: Send + Sync + 'static { } /// Bounded details of one confirmed shadow mismatch warning. -#[derive(Debug, Clone, PartialEq, Eq)] +/// +/// A field group is `None` when its leaf is off. Inside a requested group a +/// `None` field is unavailable: [`logging_outcome`](Self::logging_outcome) +/// names the failure, or, without one, the value was absent. +#[derive(Debug, Clone, PartialEq)] pub struct ShadowMismatchDetails { /// The instance namespace of the mismatching identity. pub namespace: Arc, @@ -430,11 +462,46 @@ pub struct ShadowMismatchDetails { pub use_case: Arc, /// The entity type of the mismatching identity. pub key_type: Arc, + /// Age of the cached value at the verdict, as [`Event::ShadowValueAge`] + /// records it. + pub cached_value_age_seconds: f64, /// The logical key, bounded to 2 KiB of UTF-8. pub cache_key: String, - /// JSON preview of the cached value, bounded to 8 KiB; absent when unavailable. + /// The difference fields; `None` when `diff` is off or construction failed. + pub diff: Option, + /// The JSON of both values; `None` when `value` is off or construction failed. + pub values: Option, + /// Why requested fields are unavailable; `None` when every one was built. + /// Only [`ShadowMismatchLoggingOutcome::HookError`] and + /// [`ShadowMismatchLoggingOutcome::RenderError`] appear here. + pub logging_outcome: Option, +} + +/// The difference fields of a confirmed-mismatch warning, present when +/// `mismatch_logging.diff` is above [`MismatchDiff::Off`](crate::policy::MismatchDiff::Off). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShadowMismatchDiffDetails { + /// The effective level: [`Paths`](crate::policy::MismatchDiff::Paths) or + /// [`Values`](crate::policy::MismatchDiff::Values). + pub level: crate::policy::MismatchDiff, + /// RFC 6901 JSON Pointers of the differing fields in walk order, at most + /// 64, each bounded to 512 bytes; `None` when the hook or walk failed. + pub mismatch_paths: Option>, + /// The number of differences before the cut to 64; `None` with `mismatch_paths`. + pub mismatch_path_count: Option, + /// At level `Values`, the entries as a JSON array of `{path, cached, + /// source}` objects, each side bounded to 1 KiB and the whole to 8 KiB; + /// `None` at level `Paths` and when unavailable. + pub diff_json: Option, +} + +/// The whole-value fields of a confirmed-mismatch warning, present when +/// `mismatch_logging.value` is on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShadowMismatchValueDetails { + /// JSON of the decoded cached value, bounded to 8 KiB; `None` when unavailable. pub cached_value_json: Option, - /// JSON preview of the source value, bounded to 8 KiB; absent when unavailable. + /// JSON of the source value, bounded to 8 KiB; `None` when unavailable. pub source_value_json: Option, } @@ -483,7 +550,7 @@ pub enum LogEvent { /// A shadow job's clean-miss fill failed while preparing or writing the payload. ShadowFillFailed(BoxError), /// A confirmed shadow mismatch, logged only when the resolved policy's - /// `log_mismatches` is true. + /// mismatch-logging `value` is on or its `diff` is above `off`. ShadowMismatch(ShadowMismatchDetails), /// An explicit invalidation failed; the same error is returned to the caller. InvalidationFailed(BoxError), @@ -540,13 +607,30 @@ impl fmt::Display for LogEvent { write!(f, "Error populating Redis from DialCache shadow work: {e}") } LogEvent::ShadowMismatch(d) => { - write!(f, "DialCache shadow validation mismatch: namespace={} useCase={} keyType={} cacheKey={}", - d.namespace, d.use_case, d.key_type, d.cache_key)?; - if let Some(value) = &d.cached_value_json { - write!(f, " cachedValue={value}")?; + write!(f, "DialCache shadow validation mismatch: namespace={} useCase={} keyType={} cachedValueAgeSeconds={} cacheKey={}", + d.namespace, d.use_case, d.key_type, d.cached_value_age_seconds, d.cache_key)?; + // Unavailable fields are left out; loggingOutcome says why. + if let Some(diff) = &d.diff { + if let (Some(paths), Some(count)) = + (&diff.mismatch_paths, diff.mismatch_path_count) + { + let paths = serde_json::to_string(paths).map_err(|_| fmt::Error)?; + write!(f, " mismatchPaths={paths} mismatchPathCount={count}")?; + } + if let Some(json) = &diff.diff_json { + write!(f, " diffJson={json}")?; + } + } + if let Some(values) = &d.values { + if let Some(value) = &values.cached_value_json { + write!(f, " cachedValue={value}")?; + } + if let Some(value) = &values.source_value_json { + write!(f, " sourceValue={value}")?; + } } - if let Some(value) = &d.source_value_json { - write!(f, " sourceValue={value}")?; + if let Some(outcome) = d.logging_outcome { + write!(f, " loggingOutcome={outcome}")?; } Ok(()) } diff --git a/rust/src/operation.rs b/rust/src/operation.rs index 4e09a280..152c3b07 100644 --- a/rust/src/operation.rs +++ b/rust/src/operation.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use futures::future::BoxFuture; use serde::de::DeserializeOwned; use serde::Serialize; +use serde_json::Value; use crate::codec::{Codec, JsonCodec, Payload}; use crate::error::{BoxError, Error}; @@ -14,6 +15,7 @@ use crate::limits::DEFAULT_FALLBACK_TIMEOUT_MS; use crate::local::StoredValue; use crate::policy::Policy; use crate::scope::Scope; +use crate::shadow_log::{JsonRender, ValueRenderer}; /// The source deadline of one operation. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -43,10 +45,40 @@ impl SourceBudget { pub type RecoveryPredicate = Arc bool + Send + Sync>; /// Application equality used by shadow validation. pub type Comparator = Arc bool + Send + Sync>; -/// Produces a bounded textual preview of a value for mismatch warnings. -/// Runs through [`Runtime::spawn_blocking`](crate::Runtime::spawn_blocking) -/// after mismatch confirmation; callbacks may run on a CPU worker thread. +/// Renders the text a confirmed-mismatch warning logs for a value when +/// `mismatch_logging.value` is on, in place of the value's JSON. `None`, or a +/// panic, logs the field as unavailable with a `render_error` outcome. A +/// preview you set replaces the JSON for the whole warning, so the built-in +/// difference computation never reads what it leaves out: difference fields +/// then come only from a [`MismatchDiffHook`]. Runs through +/// [`Runtime::spawn_blocking`](crate::Runtime::spawn_blocking) after mismatch +/// confirmation; callbacks may run on a CPU worker thread. pub type Preview = Arc Option + Send + Sync>; +/// Replaces the built-in difference computation of confirmed-mismatch +/// warnings for one operation: receives the compared cached and source values +/// and returns the differences. Like the comparator it must be deterministic, +/// side-effect-free and bounded; a panic logs the difference fields as +/// unavailable with a `hook_error` outcome and never falls back to the +/// built-in computation. Runs through +/// [`Runtime::spawn_blocking`](crate::Runtime::spawn_blocking) after mismatch +/// confirmation, and only when `mismatch_logging.diff` is above `off`. +pub type MismatchDiffHook = Arc Vec + Send + Sync>; + +/// One difference in a confirmed-mismatch warning: an RFC 6901 JSON Pointer +/// into the compared value and, optionally, what each side holds there. A +/// side is `None` when the member exists on the other side only, or when the +/// hook reports where the values differ but not what they hold. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct MismatchEntry { + /// The pointer, such as `/profile/email`; `""` is the whole value. + pub path: String, + /// The cached side at the pointer. + #[serde(skip_serializing_if = "Option::is_none")] + pub cached: Option, + /// The source side at the pointer. + #[serde(skip_serializing_if = "Option::is_none")] + pub source: Option, +} /// One inline cached call: its identity, static policy, budget and codecs. /// @@ -67,8 +99,59 @@ pub struct Operation { /// Per-operation stale-recovery classifier. `None` defers to the /// instance's, which by default admits only the source deadline error. pub should_recover: Option, - /// Renders bounded value previews for mismatch warnings; `None` logs none. + /// Renders the value fields of mismatch warnings; see [`Preview`]. + /// [`new`](Operation::new) sets the values' JSON rendering, which also + /// feeds the built-in difference computation. A closure you set replaces + /// that JSON for the whole warning, so difference fields then come only + /// from a [`mismatch_diff`](Self::mismatch_diff) hook. `None` logs no + /// values and leaves the operation without a JSON rendering, as one built + /// with [`with_codec`](Operation::with_codec) is: a requested value field + /// is unavailable with a `render_error` outcome, and so are difference + /// fields without a hook. pub preview: Option>, + /// Replaces the built-in difference computation of mismatch warnings. + /// `None` runs the built-in walk over the values' JSON, which only an + /// operation built with [`new`](Operation::new) that keeps its default + /// [`preview`](Self::preview) has. + pub mismatch_diff: Option>, + /// The JSON rendering captured where `T: Serialize` is known, and the + /// default preview that stands for it. + pub(crate) json: Option>, +} + +/// The JSON rendering of an operation built where `T: Serialize` is known, +/// with the default [`Preview`] installed for it. The rendering applies while +/// `Operation::preview` still holds that preview, recognized by `Arc` +/// identity: a closure set in its place replaces the JSON, and `None` removes +/// it. +pub(crate) struct JsonDefault { + render: JsonRender, + preview: Preview, +} + +impl JsonDefault { + pub(crate) fn new() -> Self { + JsonDefault { + render: JsonRender::of::(), + preview: Arc::new(crate::preview::json_preview::), + } + } +} + +impl JsonDefault { + /// The default preview, to install as `Operation::preview`. + pub(crate) fn preview(&self) -> Preview { + self.preview.clone() + } +} + +impl Clone for JsonDefault { + fn clone(&self) -> Self { + JsonDefault { + render: self.render, + preview: self.preview.clone(), + } + } } impl Clone for Operation { @@ -81,6 +164,8 @@ impl Clone for Operation { comparator: self.comparator.clone(), should_recover: self.should_recover.clone(), preview: self.preview.clone(), + mismatch_diff: self.mismatch_diff.clone(), + json: self.json.clone(), } } } @@ -97,8 +182,13 @@ impl std::fmt::Debug for Operation { impl Operation { /// An operation using the JSON codec, `PartialEq` shadow comparison and - /// JSON mismatch previews. + /// the values' JSON for mismatch warnings: its default + /// [`preview`](Self::preview) logs that JSON as the value fields, and the + /// built-in difference computation walks it. The JSON rendering stays + /// when the codec or comparator is replaced afterwards; a preview you set + /// replaces it, and `None` in place of the default removes it. pub fn new(identity: Identity) -> Self { + let json = JsonDefault::new(); Operation { identity, policy: Policy::default(), @@ -106,14 +196,21 @@ impl Operat codec: Arc::new(JsonCodec), comparator: Arc::new(|a: &T, b: &T| a == b), should_recover: None, - preview: Some(Arc::new(crate::preview::json_preview::)), + preview: Some(json.preview()), + mismatch_diff: None, + json: Some(json), } } } impl Operation { /// An operation with an explicit codec and comparator, for values that - /// are not `serde` JSON values. Mismatch previews are absent unless set. + /// are not `serde` JSON values. Mismatch warnings have no JSON rendering: + /// their value fields need a [`preview`](Self::preview) and their + /// difference fields a [`mismatch_diff`](Self::mismatch_diff) hook, and a + /// field logged without one is unavailable with a `render_error` outcome. + /// A JSON renderer for such operations is tracked in + /// . pub fn with_codec( identity: Identity, codec: Arc>, @@ -127,6 +224,8 @@ impl Operation { comparator: Arc::new(comparator), should_recover: None, preview: None, + mismatch_diff: None, + json: None, } } @@ -152,7 +251,10 @@ impl Operation { self } - /// Set the mismatch-warning preview; `None` from the closure omits the value. + /// Set the text mismatch warnings log for each value in place of its + /// JSON; `None` from the closure logs the field as unavailable. Difference + /// fields then come only from a [`mismatch_diff`](Self::mismatch_diff) + /// hook. pub fn preview( mut self, preview: impl Fn(&T) -> Option + Send + Sync + 'static, @@ -161,10 +263,32 @@ impl Operation { self } + /// Replace the built-in difference computation of mismatch warnings; see + /// [`MismatchDiffHook`]. + pub fn mismatch_diff( + mut self, + hook: impl Fn(&T, &T) -> Vec + Send + Sync + 'static, + ) -> Self { + self.mismatch_diff = Some(Arc::new(hook)); + self + } + pub(crate) fn erase(self) -> (Identity, Arc) { let codec = self.codec; let comparator = self.comparator; - let preview = self.preview; + // The default preview stands for the JSON rendering. Any other + // preview replaces it, and `None` leaves the operation without one. + let value_renderer = match (self.preview, self.json) { + (Some(preview), Some(json)) if Arc::ptr_eq(&preview, &json.preview) => { + Some(ValueRenderer::Json(json.render)) + } + (preview, _) => preview.map(|preview| { + ValueRenderer::Preview(Arc::new(move |value: &StoredValue| { + value.downcast_ref::().and_then(|v| preview(v)) + })) + }), + }; + let mismatch_diff = self.mismatch_diff; let metadata = OperationMetadata { value_type: TypeId::of::(), policy: self.policy, @@ -177,10 +301,16 @@ impl Operation { Ok(comparator(a, b)) }), should_recover: self.should_recover, - preview: preview.map(|preview| { - Arc::new(move |value: &StoredValue| { - value.downcast_ref::().and_then(|v| preview(v)) - }) as Arc Option + Send + Sync> + value_renderer, + mismatch_diff: mismatch_diff.map(|hook| { + Arc::new(move |cached: &StoredValue, source: &StoredValue| { + // The comparator already accepted both values as `T`. + let typed = "compared values have the operation's type"; + hook( + cached.downcast_ref::().expect(typed), + source.downcast_ref::().expect(typed), + ) + }) as ErasedMismatchDiff }), }; (self.identity, Arc::new(metadata)) @@ -193,6 +323,8 @@ pub(crate) type ErasedLoad = pub(crate) type ErasedCompare = Arc Result + Send + Sync>; pub(crate) type ErasedPreview = Arc Option + Send + Sync>; +pub(crate) type ErasedMismatchDiff = + Arc Vec + Send + Sync>; pub(crate) type IdentityProvider = Arc Result + Send + Sync>; /// Immutable adapters and policy, shared by every registered invocation. @@ -203,7 +335,10 @@ pub(crate) struct OperationMetadata { pub codec: Arc, pub compare: ErasedCompare, pub should_recover: Option, - pub preview: Option, + /// The values' JSON while the operation keeps its default preview, the + /// preview set in its place, or `None` without either. + pub value_renderer: Option, + pub mismatch_diff: Option, } /// The type-erased operation the engine executes. diff --git a/rust/src/policy.rs b/rust/src/policy.rs index 78c890a7..f94d3617 100644 --- a/rust/src/policy.rs +++ b/rust/src/policy.rs @@ -10,8 +10,64 @@ use crate::observe::DisabledReason; pub struct ShadowPolicy { /// Independent stable cohort percentage in `[0, 100]`. Omitted and zero disable shadow work. pub ramp: Option, - /// Emit one warning per confirmed mismatch. Defaults to false. + /// Shorthand for the whole [`mismatch_logging`](Self::mismatch_logging) + /// group: `true` stands for `value` on with [`MismatchDiff::Values`], + /// `false` for both leaves off. Leaves set in `mismatch_logging` override + /// what it expands to, before runtime overlays merge leaf by leaf. pub log_mismatches: Option, + /// Content of the warning emitted per confirmed mismatch. Omitted leaves + /// default off. + pub mismatch_logging: Option, +} + +/// Content of the one warning emitted per confirmed shadow mismatch. Both +/// leaves default off, and the warning is emitted when either is on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct MismatchLogging { + /// Log the JSON of both compared values. Defaults to false. + pub value: Option, + /// How much of the structural difference to log. Defaults to + /// [`MismatchDiff::Off`]. + pub diff: Option, +} + +/// How much of the structural difference a confirmed-mismatch warning logs. +/// The levels are ordered: each logs everything the previous one does. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] +pub enum MismatchDiff { + /// No difference fields. + #[default] + Off, + /// Where the values differ: `mismatchPaths` and `mismatchPathCount`. + Paths, + /// The paths plus what each side holds there: `diffJson`. + Values, +} + +impl MismatchDiff { + /// The level as the JSON configuration spells it: `off`, `paths` or `values`. + pub fn as_str(self) -> &'static str { + match self { + MismatchDiff::Off => "off", + MismatchDiff::Paths => "paths", + MismatchDiff::Values => "values", + } + } + + fn from_json(value: &Value) -> Option { + match value.as_str()? { + "off" => Some(MismatchDiff::Off), + "paths" => Some(MismatchDiff::Paths), + "values" => Some(MismatchDiff::Values), + _ => None, + } + } +} + +impl std::fmt::Display for MismatchDiff { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } } /// Static caching policy of one operation. Omitted leaves use library @@ -61,7 +117,11 @@ impl Policy { remote_ramp: Some(0.0), shadow: Some(ShadowPolicy { ramp: Some(0.0), - log_mismatches: Some(false), + log_mismatches: None, + mismatch_logging: Some(MismatchLogging { + value: Some(false), + diff: Some(MismatchDiff::Off), + }), }), ..Policy::default() } @@ -199,6 +259,23 @@ impl Policy { })?; parsed.log_mismatches = Some(flag); } + if let Some(group) = shadow.get("mismatchLogging") { + let group = policy_object(group, "shadow mismatchLogging config")?; + let mut leaves = MismatchLogging::default(); + if let Some(leaf) = group.get("value") { + let flag = leaf.as_bool().ok_or_else(|| { + PolicyError("shadow.mismatchLogging.value must be boolean".to_string()) + })?; + leaves.value = Some(flag); + } + if let Some(leaf) = group.get("diff") { + let level = MismatchDiff::from_json(leaf).ok_or_else(|| { + PolicyError("invalid static shadow.mismatchLogging.diff".to_string()) + })?; + leaves.diff = Some(level); + } + parsed.mismatch_logging = Some(leaves); + } policy.shadow = Some(parsed); } policy.validate()?; @@ -248,6 +325,16 @@ impl Policy { if let Some(flag) = shadow.log_mismatches { leaves.insert("logMismatches".to_string(), Value::Bool(flag)); } + if let Some(group) = shadow.mismatch_logging { + let mut written = Map::new(); + if let Some(flag) = group.value { + written.insert("value".to_string(), Value::Bool(flag)); + } + if let Some(level) = group.diff { + written.insert("diff".to_string(), Value::from(level.as_str())); + } + leaves.insert("mismatchLogging".to_string(), Value::Object(written)); + } object.insert("shadow".to_string(), Value::Object(leaves)); } Value::Object(object) @@ -384,18 +471,33 @@ pub struct ResolvedShadow { pub enabled: bool, /// The shadow cohort percentage; zero when omitted or invalid. pub ramp: f64, - /// Whether a confirmed mismatch logs a warning; `false` when the flag - /// was omitted or malformed. - pub log_mismatches: bool, + /// Whether a confirmed-mismatch warning carries the JSON of both values: + /// the effective `mismatchLogging.value` leaf after the shorthand + /// expanded, `false` when omitted or invalid. + pub mismatch_value: bool, + /// How much of the difference a confirmed-mismatch warning carries: the + /// effective `mismatchLogging.diff` leaf, [`MismatchDiff::Off`] when + /// omitted or invalid. + pub mismatch_diff: MismatchDiff, /// The shadow ramp leaf was present but invalid: shadow work is off and a /// `config_resolution` error is emitted on the remote layer whenever a /// shadow job would otherwise have been scheduled (a remote hit or a /// ramped-down remote). pub config_error: bool, - /// Recorded only if a job is admitted. + /// A mismatch-logging leaf or the `logMismatches` shorthand was invalid + /// and acts off. One `config_resolution` covers all of them, recorded only + /// if a job is admitted. pub logging_config_error: bool, } +impl ResolvedShadow { + /// Whether a confirmed mismatch emits a warning: `value` is on or `diff` + /// is above [`MismatchDiff::Off`]. + pub fn logs_mismatches(&self) -> bool { + self.mismatch_value || self.mismatch_diff != MismatchDiff::Off + } +} + /// The captured policy of one enabled invocation. #[derive(Debug, Clone, PartialEq)] pub struct ResolvedPolicy { @@ -505,15 +607,10 @@ pub fn resolve_policy( None => shadow.config_error = true, } } - if let Some(leaf) = leaves.get("logMismatches") { - match leaf.as_bool() { - Some(flag) => shadow.log_mismatches = flag, - None => { - shadow.log_mismatches = false; - shadow.logging_config_error = true; - } - } - } + let logging = resolve_mismatch_logging(leaves); + shadow.mismatch_value = logging.value; + shadow.mismatch_diff = logging.diff; + shadow.logging_config_error = logging.invalid; } Ok(ResolvedPolicy { @@ -537,7 +634,6 @@ fn merge_overlay(merged: &mut Map, overlay: &Value) -> Result<(), for (name, leaves) in [ ("ttlSec", &["local", "remote"][..]), ("ramp", &["local", "remote"][..]), - ("shadow", &["ramp", "logMismatches"][..]), ] { let Some(incoming) = overlay.get(name) else { continue; @@ -554,6 +650,17 @@ fn merge_overlay(merged: &mut Map, overlay: &Value) -> Result<(), } merged.insert(name.to_string(), Value::Object(output)); } + if let Some(incoming) = overlay.get("shadow") { + let incoming = policy_object(incoming, "shadow")?; + let inherited = match merged.get("shadow") { + Some(Value::Object(existing)) => existing.clone(), + _ => Map::new(), + }; + merged.insert( + "shadow".to_string(), + Value::Object(merge_shadow(&inherited, incoming)?), + ); + } for name in [ "requestLocal", "coalesce", @@ -567,6 +674,109 @@ fn merge_overlay(merged: &mut Map, overlay: &Value) -> Result<(), Ok(()) } +/// Merge one runtime `shadow` group over the inherited one. `ramp` merges as +/// a leaf. Each layer's `logMismatches` shorthand expands to the leaves it +/// stands for before the leaf-wise merge, so a runtime shorthand replaces both +/// inherited leaves while a leaf written beside it still wins within its own +/// layer. A non-object runtime `mismatchLogging` fails the merge. +fn merge_shadow( + inherited: &Map, + incoming: &Map, +) -> Result, PolicyError> { + let mut output = Map::new(); + if let Some(ramp) = incoming.get("ramp").or_else(|| inherited.get("ramp")) { + output.insert("ramp".to_string(), ramp.clone()); + } + if let Some(group) = incoming.get("mismatchLogging") { + policy_object(group, "shadow mismatchLogging config")?; + } + let base = expand_mismatch_logging(inherited); + let written = expand_mismatch_logging(incoming); + // Only the runtime layer can carry a malformed shorthand: a static one is + // a typed boolean. It stays raw, so resolution records the one + // `config_resolution` it owes; its expanded leaves already won the merge. + if written.invalid { + if let Some(shorthand) = incoming.get("logMismatches") { + output.insert("logMismatches".to_string(), shorthand.clone()); + } + } + let mut group = Map::new(); + if let Some(value) = written.value.or(base.value) { + group.insert("value".to_string(), value); + } + if let Some(diff) = written.diff.or(base.diff) { + group.insert("diff".to_string(), diff); + } + if !group.is_empty() { + output.insert("mismatchLogging".to_string(), Value::Object(group)); + } + Ok(output) +} + +/// The mismatch-logging leaves one shadow group writes, before validation; +/// `None` is unwritten. `invalid` marks a malformed shorthand. +#[derive(Default)] +struct MismatchLeaves { + value: Option, + diff: Option, + invalid: bool, +} + +/// Expand one shadow group's `logMismatches` shorthand into the leaves it +/// stands for, `true` as value on with diff `values` and `false` as both off, +/// then let the leaves written in the same group's `mismatchLogging` override +/// them. A malformed shorthand expands to both leaves off and is marked +/// invalid. +fn expand_mismatch_logging(shadow: &Map) -> MismatchLeaves { + let expanded = |value: bool, diff: MismatchDiff, invalid: bool| MismatchLeaves { + value: Some(Value::Bool(value)), + diff: Some(Value::from(diff.as_str())), + invalid, + }; + let mut leaves = match shadow.get("logMismatches") { + None => MismatchLeaves::default(), + Some(Value::Bool(true)) => expanded(true, MismatchDiff::Values, false), + Some(Value::Bool(false)) => expanded(false, MismatchDiff::Off, false), + Some(_) => expanded(false, MismatchDiff::Off, true), + }; + if let Some(group) = shadow.get("mismatchLogging").and_then(Value::as_object) { + if let Some(value) = group.get("value") { + leaves.value = Some(value.clone()); + } + if let Some(diff) = group.get("diff") { + leaves.diff = Some(diff.clone()); + } + } + leaves +} + +/// The effective mismatch-logging leaves of one merged shadow group. +struct MismatchLoggingResolution { + value: bool, + diff: MismatchDiff, + invalid: bool, +} + +/// Resolve each leaf independently: an omitted leaf is off, an invalid leaf +/// acts off, and `invalid` reports that the group owes one +/// `config_resolution`, however many of its inputs were invalid. +fn resolve_mismatch_logging(shadow: &Map) -> MismatchLoggingResolution { + let leaves = expand_mismatch_logging(shadow); + let value = match &leaves.value { + None => Some(false), + Some(leaf) => leaf.as_bool(), + }; + let diff = match &leaves.diff { + None => Some(MismatchDiff::Off), + Some(leaf) => MismatchDiff::from_json(leaf), + }; + MismatchLoggingResolution { + value: value.unwrap_or(false), + diff: diff.unwrap_or(MismatchDiff::Off), + invalid: leaves.invalid || value.is_none() || diff.is_none(), + } +} + fn resolve_layer( ttls: &Map, ramps: &Map, @@ -708,7 +918,7 @@ mod tests { assert_eq!(r.remote_read_timeout_ms, 30); assert!(r.shadow.enabled); assert_eq!(r.shadow.ramp, 100.0); - assert!(!r.shadow.log_mismatches); + assert!(!r.shadow.logs_mismatches()); assert!(!r.shadow.config_error); assert!(!r.shadow.logging_config_error); @@ -733,7 +943,8 @@ mod tests { assert!(inherit.remote.enabled); assert_eq!(inherit.remote_read_timeout_ms, 30); assert!(inherit.shadow.enabled); - assert!(inherit.shadow.log_mismatches); + assert!(inherit.shadow.mismatch_value); + assert_eq!(inherit.shadow.mismatch_diff, MismatchDiff::Values); } } @@ -784,6 +995,15 @@ mod tests { json!({"shadow": {"ramp": "50"}}), json!({"shadow": {"logMismatches": null}}), json!({"shadow": {"logMismatches": 1}}), + json!({"shadow": {"mismatchLogging": null}}), + json!({"shadow": {"mismatchLogging": []}}), + json!({"shadow": {"mismatchLogging": true}}), + json!({"shadow": {"mismatchLogging": {"value": null}}}), + json!({"shadow": {"mismatchLogging": {"value": "true"}}}), + json!({"shadow": {"mismatchLogging": {"diff": null}}}), + json!({"shadow": {"mismatchLogging": {"diff": "full"}}}), + json!({"shadow": {"mismatchLogging": {"diff": true}}}), + json!({"shadow": {"mismatchLogging": {"diff": "Paths"}}}), ]; for config in invalid { assert!( @@ -869,17 +1089,21 @@ mod tests { .is_err()); let nan_shadow = ShadowPolicy { ramp: Some(f64::NAN), - log_mismatches: None, + ..ShadowPolicy::default() }; assert!(Policy::default().shadow(nan_shadow).validate().is_err()); let high_shadow = ShadowPolicy { ramp: Some(100.5), - log_mismatches: None, + ..ShadowPolicy::default() }; assert!(Policy::default().shadow(high_shadow).validate().is_err()); let flag_only = ShadowPolicy { - ramp: None, log_mismatches: Some(true), + mismatch_logging: Some(MismatchLogging { + value: Some(false), + diff: Some(MismatchDiff::Paths), + }), + ..ShadowPolicy::default() }; assert!(Policy::default().shadow(flag_only).validate().is_ok()); assert!(Policy::enabled(60).validate().is_ok()); @@ -910,7 +1134,11 @@ mod tests { "coalesce": false, "staleOnErrorMaxAgeSec": 120, "remoteReadTimeoutMs": 25, - "shadow": {"ramp": 3.5, "logMismatches": true}, + "shadow": { + "ramp": 3.5, + "logMismatches": true, + "mismatchLogging": {"value": false, "diff": "paths"}, + }, }); let policy = Policy::from_json(&source).unwrap(); assert_eq!( @@ -926,7 +1154,11 @@ mod tests { remote_read_timeout_ms: Some(25), shadow: Some(ShadowPolicy { ramp: Some(3.5), - log_mismatches: Some(true) + log_mismatches: Some(true), + mismatch_logging: Some(MismatchLogging { + value: Some(false), + diff: Some(MismatchDiff::Paths), + }), }), } ); @@ -969,8 +1201,8 @@ mod tests { assert_eq!( Policy::default() .shadow(ShadowPolicy { - ramp: None, - log_mismatches: Some(false) + log_mismatches: Some(false), + ..ShadowPolicy::default() }) .to_json(), json!({ @@ -978,6 +1210,22 @@ mod tests { "shadow": {"logMismatches": false}, }) ); + // Only the set mismatch-logging leaves appear in their group. + assert_eq!( + Policy::default() + .shadow(ShadowPolicy { + mismatch_logging: Some(MismatchLogging { + value: None, + diff: Some(MismatchDiff::Values), + }), + ..ShadowPolicy::default() + }) + .to_json(), + json!({ + "ttlSec": {}, "ramp": {}, + "shadow": {"mismatchLogging": {"diff": "values"}}, + }) + ); // Zero recovery age is a present leaf, not an omission. assert_eq!( Policy::default() @@ -1003,7 +1251,7 @@ mod tests { "ramp": {"local": 0, "remote": 0}, "requestLocal": false, "staleOnErrorMaxAgeSec": 0, - "shadow": {"ramp": 0, "logMismatches": false}, + "shadow": {"ramp": 0, "mismatchLogging": {"value": false, "diff": "off"}}, }) ); } @@ -1034,7 +1282,7 @@ mod tests { assert_eq!(alone.remote.reason, Some(DisabledReason::PolicyDisabled)); assert!(!alone.stale_on_error_config_error); assert!(!alone.shadow.enabled && !alone.shadow.config_error); - assert!(!alone.shadow.log_mismatches && !alone.shadow.logging_config_error); + assert!(!alone.shadow.logs_mismatches() && !alone.shadow.logging_config_error); // As an overlay it ramps every inherited path to zero instead of relying on omission. let base = Policy::enabled(60) @@ -1043,6 +1291,10 @@ mod tests { .shadow(ShadowPolicy { ramp: Some(100.0), log_mismatches: Some(true), + mismatch_logging: Some(MismatchLogging { + value: Some(true), + diff: Some(MismatchDiff::Values), + }), }); let killed = resolved(&base, Some(RuntimePolicy::from(Policy::disabled()).0)); assert!(!killed.request_local); @@ -1292,7 +1544,7 @@ mod tests { assert_eq!(r.shadow.ramp, 100.0); assert!(!r.shadow.config_error); assert!(r.shadow.logging_config_error); - assert!(!r.shadow.log_mismatches); + assert!(!r.shadow.logs_mismatches()); let r = resolved(&base, Some(json!({"shadow": {"ramp": null}}))); assert!(r.shadow.config_error); @@ -1317,7 +1569,7 @@ mod tests { assert!( logging.shadow.enabled && logging.shadow.logging_config_error - && !logging.shadow.log_mismatches + && !logging.shadow.logs_mismatches() ); } @@ -1331,6 +1583,7 @@ mod tests { .shadow(ShadowPolicy { ramp: Some(100.0), log_mismatches: Some(true), + mismatch_logging: None, }); // Omitted leaves inherit the operation leaf; an empty overlay changes nothing. @@ -1344,7 +1597,8 @@ mod tests { assert!(inherited.request_local && !inherited.coalesce); assert_eq!(inherited.remote_read_timeout_ms, 40); assert_eq!(inherited.stale_on_error_max_age_ms, 30_000); - assert!(inherited.shadow.enabled && inherited.shadow.log_mismatches); + assert!(inherited.shadow.enabled && inherited.shadow.mismatch_value); + assert_eq!(inherited.shadow.mismatch_diff, MismatchDiff::Values); // A present leaf replaces the operation leaf, sparsely. let patched = resolved( @@ -1401,10 +1655,12 @@ mod tests { // Shadow leaves merge independently of serving leaves. let shadow = resolved(&base, Some(json!({"shadow": {"logMismatches": false}}))); - assert!(shadow.shadow.enabled && !shadow.shadow.log_mismatches); + assert!(shadow.shadow.enabled && !shadow.shadow.logs_mismatches()); let shadow = resolved(&base, Some(json!({"shadow": {"ramp": 0}}))); assert!( - !shadow.shadow.enabled && shadow.shadow.log_mismatches && !shadow.shadow.config_error + !shadow.shadow.enabled + && shadow.shadow.logs_mismatches() + && !shadow.shadow.config_error ); assert!(shadow.local.enabled && shadow.remote.enabled); } @@ -1431,7 +1687,7 @@ mod tests { ), ); assert_eq!(r.stale_on_error_max_age_ms, 7_000); - assert!(r.shadow.enabled && !r.shadow.log_mismatches); + assert!(r.shadow.enabled && !r.shadow.logs_mismatches()); // A runtime static-policy overlay (via From) behaves like its JSON form. let overlay = RuntimePolicy::from(Policy::default().local_ttl_sec(9)); let r = resolve_policy( @@ -1604,4 +1860,339 @@ mod tests { PolicyError("invalid runtime remoteReadTimeoutMs".to_string()) ); } + + fn shadow_logging(base: &Policy, overlay: Option) -> (bool, MismatchDiff, bool) { + let shadow = resolved(base, overlay).shadow; + ( + shadow.mismatch_value, + shadow.mismatch_diff, + shadow.logging_config_error, + ) + } + + fn shadow_base(log_mismatches: Option, group: Option) -> Policy { + Policy::default().shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches, + mismatch_logging: group, + }) + } + + #[test] + fn mismatch_logging_resolves_the_recorded_four_row_table() { + use MismatchDiff::{Off, Paths, Values}; + let leaves = |value, diff| Some(MismatchLogging { value, diff }); + // { logMismatches: true, mismatchLogging: { value: false } }, in either layer. + let row = shadow_base(Some(true), leaves(Some(false), None)); + assert_eq!(shadow_logging(&row, None), (false, Values, false)); + assert_eq!( + shadow_logging( + &shadow_base(None, None), + Some( + json!({"shadow": {"logMismatches": true, "mismatchLogging": {"value": false}}}) + ) + ), + (false, Values, false) + ); + // { logMismatches: false, mismatchLogging: { diff: "paths" } }, in either layer. + let row = shadow_base(Some(false), leaves(None, Some(Paths))); + assert_eq!(shadow_logging(&row, None), (false, Paths, false)); + assert_eq!( + shadow_logging( + &shadow_base(None, None), + Some( + json!({"shadow": {"logMismatches": false, "mismatchLogging": {"diff": "paths"}}}) + ) + ), + (false, Paths, false) + ); + // A runtime shorthand replaces both inherited leaves. + let paths = shadow_base(None, leaves(None, Some(Paths))); + assert_eq!( + shadow_logging(&paths, Some(json!({"shadow": {"logMismatches": false}}))), + (false, Off, false) + ); + let value_off = shadow_base(None, leaves(Some(false), None)); + assert_eq!( + shadow_logging(&value_off, Some(json!({"shadow": {"logMismatches": true}}))), + (true, Values, false) + ); + } + + #[test] + fn mismatch_logging_leaves_merge_after_each_layer_expands() { + use MismatchDiff::{Off, Paths, Values}; + let leaves = |value, diff| Some(MismatchLogging { value, diff }); + // An omitted runtime leaf inherits; a written one wins leaf by leaf. + let base = shadow_base(None, leaves(Some(true), Some(Paths))); + assert_eq!( + shadow_logging( + &base, + Some(json!({"shadow": {"mismatchLogging": {"diff": "values"}}})) + ), + (true, Values, false) + ); + assert_eq!( + shadow_logging( + &base, + Some(json!({"shadow": {"mismatchLogging": {"value": false}}})) + ), + (false, Paths, false) + ); + // A static shorthand is a baseline like any other pair of static leaves. + let shorthand = shadow_base(Some(true), None); + assert_eq!( + shadow_logging( + &shorthand, + Some(json!({"shadow": {"mismatchLogging": {"diff": "paths"}}})) + ), + (true, Paths, false) + ); + // An overlay without a shadow group, or with an empty one, keeps the static leaves. + for overlay in [ + json!({}), + json!({"shadow": {}}), + json!({"shadow": {"ramp": 50}}), + ] { + assert_eq!( + shadow_logging(&shorthand, Some(overlay)), + (true, Values, false) + ); + } + // The disabled() overlay turns every inherited leaf off without a diagnostic. + let all_on = shadow_base(Some(true), leaves(Some(true), Some(Values))); + assert_eq!( + shadow_logging(&all_on, Some(RuntimePolicy::from(Policy::disabled()).0)), + (false, Off, false) + ); + } + + #[test] + fn a_malformed_runtime_shorthand_stays_raw_for_one_config_resolution() { + let inherited = shadow_base( + None, + Some(MismatchLogging { + value: Some(false), + diff: Some(MismatchDiff::Values), + }), + ); + let written = json!({"logMismatches": "yes", "mismatchLogging": {"value": true}}); + let base = match inherited.to_json() { + Value::Object(object) => object["shadow"].as_object().unwrap().clone(), + _ => unreachable!(), + }; + // The malformed shorthand expands to both leaves off over the static + // leaves, the leaf written beside it still wins, and the raw value stays. + let merged = merge_shadow(&base, written.as_object().unwrap()).unwrap(); + assert_eq!( + Value::Object(merged), + json!({ + "ramp": 100, + "logMismatches": "yes", + "mismatchLogging": {"value": true, "diff": "off"}, + }) + ); + assert_eq!( + shadow_logging(&inherited, Some(json!({"shadow": written}))), + (true, MismatchDiff::Off, true) + ); + // Any number of invalid inputs owes the same single flag. + assert_eq!( + shadow_logging( + &inherited, + Some(json!({"shadow": { + "logMismatches": null, + "mismatchLogging": {"value": "yes", "diff": "full"}, + }})) + ), + (false, MismatchDiff::Off, true) + ); + } + + #[test] + fn mismatch_logging_resolves_each_leaf_independently() { + use MismatchDiff::{Off, Paths, Values}; + let cases = [ + (json!({}), (false, Off, false)), + (json!({"mismatchLogging": {}}), (false, Off, false)), + ( + json!({"mismatchLogging": {"value": true, "diff": "paths"}}), + (true, Paths, false), + ), + ( + json!({"mismatchLogging": {"value": "true", "diff": "paths"}}), + (false, Paths, true), + ), + ( + json!({"mismatchLogging": {"value": null}}), + (false, Off, true), + ), + ( + json!({"mismatchLogging": {"value": true, "diff": "full"}}), + (true, Off, true), + ), + (json!({"mismatchLogging": {"diff": 1}}), (false, Off, true)), + ( + json!({"mismatchLogging": {"diff": null}}), + (false, Off, true), + ), + ( + json!({"mismatchLogging": {"diff": "toString"}}), + (false, Off, true), + ), + ( + json!({"mismatchLogging": {"value": true, "diff": "constructor"}}), + (true, Off, true), + ), + ( + json!({"mismatchLogging": {"diff": "Values"}}), + (false, Off, true), + ), + (json!({"logMismatches": "yes"}), (false, Off, true)), + (json!({"logMismatches": null}), (false, Off, true)), + (json!({"logMismatches": 0}), (false, Off, true)), + ( + json!({"logMismatches": "yes", "mismatchLogging": {"value": true, "diff": "values"}}), + (true, Values, true), + ), + // Unknown members stay ignored, as on main. + ( + json!({"mismatchLogging": {"diff": "paths", "unknown": 1}, "unknown": true}), + (false, Paths, false), + ), + ]; + for (shadow, expected) in cases { + assert_eq!( + shadow_logging(&Policy::default(), Some(json!({"shadow": shadow.clone()}))), + expected, + "{shadow}" + ); + } + } + + #[test] + fn a_non_object_runtime_mismatch_logging_group_fails_resolution() { + for group in [ + json!(null), + json!([]), + json!(1), + json!("paths"), + json!(true), + ] { + for base in [Policy::default(), shadow_base(Some(true), None)] { + assert_eq!( + resolve( + &base, + Some(json!({"shadow": {"logMismatches": true, "mismatchLogging": group.clone()}})) + ) + .unwrap_err(), + PolicyError("DialCache shadow mismatchLogging config must be an object".to_string()), + "{group}" + ); + } + } + } + + // Port of the TypeScript merge grid: the recorded rule is written out + // without the code under test and checked over every combination. + #[test] + fn every_combination_of_shorthand_and_leaves_resolves_as_each_layer_expanded_first() { + fn layer_leaves( + shorthand: Option<&Value>, + group: Option<&Value>, + ) -> (Option, Option) { + let (mut value, mut diff) = (None, None); + if let Some(shorthand) = shorthand { + let on = shorthand == &Value::Bool(true); + value = Some(Value::Bool(on)); + diff = Some(Value::from(if on { "values" } else { "off" })); + } + if let Some(group) = group.and_then(Value::as_object) { + if let Some(leaf) = group.get("value") { + value = Some(leaf.clone()); + } + if let Some(leaf) = group.get("diff") { + diff = Some(leaf.clone()); + } + } + (value, diff) + } + let static_shorthands = [None, Some(true), Some(false)]; + let static_groups = [ + None, + Some((None, None)), + Some((Some(true), None)), + Some((Some(false), None)), + Some((None, Some(MismatchDiff::Paths))), + Some((None, Some(MismatchDiff::Off))), + Some((Some(true), Some(MismatchDiff::Values))), + ]; + let runtime_shorthands = [ + None, + Some(json!(true)), + Some(json!(false)), + Some(json!("yes")), + Some(json!(null)), + Some(json!(0)), + ]; + let runtime_groups = [ + None, + Some(json!({})), + Some(json!({"value": true})), + Some(json!({"value": false})), + Some(json!({"diff": "paths"})), + Some(json!({"diff": "values"})), + Some(json!({"value": true, "diff": "off"})), + Some(json!({"value": "yes"})), + Some(json!({"value": null})), + Some(json!({"diff": "full"})), + Some(json!({"diff": "toString"})), + ]; + let mut combinations = 0; + for static_shorthand in static_shorthands { + for static_group in static_groups { + let base = shadow_base( + static_shorthand, + static_group.map(|(value, diff)| MismatchLogging { value, diff }), + ); + let static_shadow = base.to_json()["shadow"].clone(); + let inherited = layer_leaves( + static_shadow.get("logMismatches"), + static_shadow.get("mismatchLogging"), + ); + for runtime_shorthand in &runtime_shorthands { + for runtime_group in &runtime_groups { + let mut shadow = Map::new(); + if let Some(shorthand) = runtime_shorthand { + shadow.insert("logMismatches".to_string(), shorthand.clone()); + } + if let Some(group) = runtime_group { + shadow.insert("mismatchLogging".to_string(), group.clone()); + } + let written = + layer_leaves(runtime_shorthand.as_ref(), runtime_group.as_ref()); + let value = written.0.or(inherited.0.clone()); + let diff = written.1.or(inherited.1.clone()); + let valid_value = value.as_ref().is_none_or(Value::is_boolean); + let level = diff.as_ref().and_then(MismatchDiff::from_json); + let valid_diff = diff.is_none() || level.is_some(); + let malformed = runtime_shorthand.as_ref().is_some_and(|s| !s.is_boolean()); + let expected = ( + valid_value && value == Some(Value::Bool(true)), + level.unwrap_or(MismatchDiff::Off), + malformed || !valid_value || !valid_diff, + ); + let overlay = json!({"shadow": Value::Object(shadow)}); + assert_eq!( + shadow_logging(&base, Some(overlay.clone())), + expected, + "static {static_shadow} runtime {overlay}" + ); + combinations += 1; + } + } + } + } + assert_eq!(combinations, 3 * 7 * 6 * 11); + } } diff --git a/rust/src/preview.rs b/rust/src/preview.rs index 93efe84c..5f26e12c 100644 --- a/rust/src/preview.rs +++ b/rust/src/preview.rs @@ -1,32 +1,42 @@ -//! Bounded previews for mismatch warnings. +//! Bounded previews and caps for mismatch warnings. use std::io::{self, Write}; -use serde::Serialize; - /// Maximum UTF-8 bytes of a logged cache key. pub const SHADOW_LOG_KEY_MAX_BYTES: usize = 2 * 1024; /// Maximum UTF-8 bytes of a logged value preview. pub const SHADOW_LOG_VALUE_MAX_BYTES: usize = 8 * 1024; +/// Maximum UTF-8 bytes of a logged `diff_json`. +pub const SHADOW_LOG_DIFF_MAX_BYTES: usize = 8 * 1024; +/// Most difference entries a warning lists; `mismatch_path_count` keeps the +/// uncut total. +pub const SHADOW_MISMATCH_MAX_ENTRIES: usize = 64; +/// Maximum UTF-8 bytes of one logged JSON Pointer. +pub const SHADOW_MISMATCH_PATH_MAX_BYTES: usize = 512; +/// Maximum UTF-8 bytes of one entry side's JSON in `diff_json`; a longer side +/// is logged as its clamped JSON text. +pub const SHADOW_MISMATCH_SIDE_MAX_BYTES: usize = 1024; +/// Reference tokens in the deepest pointer the built-in walk reports. +pub const SHADOW_MISMATCH_MAX_DEPTH: usize = 32; /// Marker appended to a clamped preview. pub const SHADOW_LOG_TRUNCATION_MARKER: &str = "...[truncated]"; /// Retain only the logged prefix, but consume the complete serialization so /// errors beyond the cap still suppress the preview. -struct JsonPrefix { +pub(crate) struct JsonPrefix { bytes: Vec, truncated: bool, } impl JsonPrefix { - fn new() -> Self { + pub(crate) fn new() -> Self { Self { bytes: Vec::with_capacity(SHADOW_LOG_VALUE_MAX_BYTES), truncated: false, } } - fn finish(mut self) -> String { + pub(crate) fn finish(mut self) -> String { if self.truncated { self.bytes .truncate(SHADOW_LOG_VALUE_MAX_BYTES - SHADOW_LOG_TRUNCATION_MARKER.len()); @@ -59,7 +69,9 @@ impl Write for JsonPrefix { } } -pub(crate) fn json_preview(value: &T) -> Option { +/// The streamed value-field prefix of `value`, as warnings render it: the +/// default preview of an operation built with JSON. +pub(crate) fn json_preview(value: &T) -> Option { let mut prefix = JsonPrefix::new(); serde_json::to_writer(&mut prefix, value).ok()?; Some(prefix.finish()) @@ -95,6 +107,7 @@ pub fn preview_value(json: &str) -> String { mod tests { use super::*; use serde::ser::{Error, SerializeSeq}; + use serde::Serialize; #[test] fn clamps_on_character_boundaries() { diff --git a/rust/src/prometheus.rs b/rust/src/prometheus.rs index f103a12d..19144ed4 100644 --- a/rust/src/prometheus.rs +++ b/rust/src/prometheus.rs @@ -1,6 +1,6 @@ //! Prometheus metric exporter (feature `prometheus`). //! -//! [`PrometheusObserver`] registers the nineteen DialCache collectors on a +//! [`PrometheusObserver`] registers the twenty DialCache collectors on a //! [`Registry`] under the names, help text, label sets and histogram buckets //! of the TypeScript adapter, and feeds every [`Event`] to them. //! @@ -69,7 +69,7 @@ impl CollectorSchema { } } -/// The nineteen collectors DialCache registers, in registration order, with +/// The twenty collectors DialCache registers, in registration order, with /// their names under `prefix`. pub fn schemas(prefix: &str) -> Vec { let schema = |kind: MetricKind, @@ -134,6 +134,12 @@ pub fn schemas(prefix: &str) -> Vec { "Age in seconds of the validated Redis value at DialCache shadow verdict time.", VALUE_AGE_BUCKETS, ), + schema( + MetricKind::ShadowMismatchLogging, + "shadow_mismatch_logging_counter", + "DialCache confirmed shadow-mismatch warning outcomes.", + &[], + ), schema( MetricKind::FutureTimestampOffset, "future_timestamp_offset_histogram", diff --git a/rust/src/protocol/mismatch.rs b/rust/src/protocol/mismatch.rs new file mode 100644 index 00000000..59a56b8b --- /dev/null +++ b/rust/src/protocol/mismatch.rs @@ -0,0 +1,304 @@ +//! The built-in confirmed-mismatch difference: a walk over two JSON values +//! that reports RFC 6901 pointers with the sides found there. + +use std::fmt::Write; + +use serde_json::Value; + +use crate::identity::compare_utf16; +use crate::operation::MismatchEntry; +use crate::preview::{ + clamp_utf8, SHADOW_MISMATCH_MAX_DEPTH, SHADOW_MISMATCH_MAX_ENTRIES, + SHADOW_MISMATCH_PATH_MAX_BYTES, SHADOW_MISMATCH_SIDE_MAX_BYTES, +}; + +/// The capped built-in difference list and its uncut length. +#[derive(Debug, Clone, PartialEq)] +pub struct MismatchJsonDiff { + /// Bounded entries in walk order, at most + /// [`SHADOW_MISMATCH_MAX_ENTRIES`](crate::preview::SHADOW_MISMATCH_MAX_ENTRIES). + pub entries: Vec, + /// Every difference the walk found, before the cut. + pub count: u64, +} + +/// The built-in difference computation over two JSON values. +/// +/// Objects recurse over the union of their member names in UTF-16 code-unit +/// order, and a member present on one side only is reported at its pointer +/// with that side. Arrays of equal length recurse by index; arrays of +/// different length are reported once at their own pointer. Any other +/// differing pair, roots of different kinds included, is reported at its +/// pointer with both sides, and a differing pair of containers at +/// [`SHADOW_MISMATCH_MAX_DEPTH`](crate::preview::SHADOW_MISMATCH_MAX_DEPTH) +/// reference tokens is reported whole without descending. Numbers compare by +/// value as doubles, so `1` and `1.0` are equal. Entries keep walk order and +/// are cut at +/// [`SHADOW_MISMATCH_MAX_ENTRIES`](crate::preview::SHADOW_MISMATCH_MAX_ENTRIES); +/// `count` is the uncut total. Each pointer is clamped to +/// [`SHADOW_MISMATCH_PATH_MAX_BYTES`](crate::preview::SHADOW_MISMATCH_PATH_MAX_BYTES), +/// and a side whose JSON exceeds +/// [`SHADOW_MISMATCH_SIDE_MAX_BYTES`](crate::preview::SHADOW_MISMATCH_SIDE_MAX_BYTES) +/// becomes its clamped JSON text as a string. +pub fn diff_shadow_mismatch_json(cached: &Value, source: &Value) -> MismatchJsonDiff { + let walk = walk_json(cached, source); + MismatchJsonDiff { + entries: walk + .entries + .iter() + .map(|entry| bound_entry(&entry.path, entry.cached, entry.source, true)) + .collect(), + count: walk.count, + } +} + +/// The difference between two renderings, `None` for an absent side. Equal +/// texts have no entries; one absent side is reported whole at the root. +/// `with_values` keeps the bounded sides; without it entries carry paths only. +pub(crate) fn diff_rendered( + cached: Option<&str>, + source: Option<&str>, + with_values: bool, +) -> Result { + if cached == source { + return Ok(MismatchJsonDiff { + entries: Vec::new(), + count: 0, + }); + } + let cached: Option = cached.map(serde_json::from_str).transpose()?; + let source: Option = source.map(serde_json::from_str).transpose()?; + let (cached, source) = match (&cached, &source) { + (Some(cached), Some(source)) => (cached, source), + _ => { + return Ok(MismatchJsonDiff { + entries: vec![bound_entry( + "", + cached.as_ref(), + source.as_ref(), + with_values, + )], + count: 1, + }) + } + }; + let walk = walk_json(cached, source); + Ok(MismatchJsonDiff { + entries: walk + .entries + .iter() + .map(|entry| bound_entry(&entry.path, entry.cached, entry.source, with_values)) + .collect(), + count: walk.count, + }) +} + +/// The one clamp rule for built-in and hook entries: the pointer is clamped, +/// and with `with_values` each present side is kept as JSON, or as its +/// clamped text when its rendering exceeds the side cap. +pub(crate) fn bound_entry( + path: &str, + cached: Option<&Value>, + source: Option<&Value>, + with_values: bool, +) -> MismatchEntry { + let side = |value: Option<&Value>| value.filter(|_| with_values).map(bound_side); + MismatchEntry { + path: clamp_utf8(path, SHADOW_MISMATCH_PATH_MAX_BYTES), + cached: side(cached), + source: side(source), + } +} + +fn bound_side(value: &Value) -> Value { + match serde_json::to_string(value) { + Ok(json) if json.len() > SHADOW_MISMATCH_SIDE_MAX_BYTES => { + Value::String(clamp_utf8(&json, SHADOW_MISMATCH_SIDE_MAX_BYTES)) + } + _ => value.clone(), + } +} + +/// One difference as the walk holds it, before the clamps. +struct RawEntry<'a> { + path: String, + cached: Option<&'a Value>, + source: Option<&'a Value>, +} + +/// One unescaped reference token of the pointer to the walk's position. +#[derive(Clone, Copy)] +enum Token<'a> { + Index(usize), + Name(&'a str), +} + +struct Walk<'a> { + entries: Vec>, + count: u64, + /// The pointer to the current position, as tokens, so descending costs + /// no copy of the prefix. + tokens: Vec>, +} + +impl<'a> Walk<'a> { + fn report(&mut self, cached: Option<&'a Value>, source: Option<&'a Value>) { + self.count += 1; + if self.entries.len() < SHADOW_MISMATCH_MAX_ENTRIES { + self.entries.push(RawEntry { + path: self.pointer(), + cached, + source, + }); + } + } + + /// The escaped pointer to the current position, built only as far as + /// [`SHADOW_MISMATCH_PATH_MAX_BYTES`] reads: past the cap it stops at a + /// character boundary, and that prefix clamps as the whole pointer would. + /// So a retained entry costs at most the cap, however long and deep the + /// member names above it. + fn pointer(&self) -> String { + let mut pointer = String::new(); + for token in &self.tokens { + if pointer.len() > SHADOW_MISMATCH_PATH_MAX_BYTES { + break; + } + pointer.push('/'); + match *token { + Token::Index(index) => { + let _ = write!(pointer, "{index}"); + } + Token::Name(name) => { + for c in name.chars() { + if pointer.len() > SHADOW_MISMATCH_PATH_MAX_BYTES { + break; + } + match c { + '~' => pointer.push_str("~0"), + '/' => pointer.push_str("~1"), + c => pointer.push(c), + } + } + } + } + } + pointer + } + + fn walk(&mut self, left: &'a Value, right: &'a Value) { + let kind = Kind::of(left); + if kind != Kind::of(right) || kind == Kind::Primitive { + if !primitives_equal(left, right) { + self.report(Some(left), Some(right)); + } + return; + } + if self.tokens.len() >= SHADOW_MISMATCH_MAX_DEPTH { + if !json_values_equal(left, right) { + self.report(Some(left), Some(right)); + } + return; + } + match (left, right) { + (Value::Array(left_items), Value::Array(right_items)) => { + if left_items.len() != right_items.len() { + self.report(Some(left), Some(right)); + return; + } + for (index, (l, r)) in left_items.iter().zip(right_items).enumerate() { + self.tokens.push(Token::Index(index)); + self.walk(l, r); + self.tokens.pop(); + } + } + (Value::Object(left_members), Value::Object(right_members)) => { + let mut names: Vec<&String> = left_members.keys().collect(); + names.extend( + right_members + .keys() + .filter(|name| !left_members.contains_key(*name)), + ); + names.sort_by(|a, b| compare_utf16(a, b)); + for name in names { + self.tokens.push(Token::Name(name)); + match (left_members.get(name), right_members.get(name)) { + (Some(l), Some(r)) => self.walk(l, r), + (l, r) => self.report(l, r), + } + self.tokens.pop(); + } + } + _ => unreachable!("both values share a container kind"), + } + } +} + +fn walk_json<'a>(cached: &'a Value, source: &'a Value) -> Walk<'a> { + let mut walk = Walk { + entries: Vec::new(), + count: 0, + tokens: Vec::new(), + }; + walk.walk(cached, source); + walk +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Kind { + Array, + Object, + Primitive, +} + +impl Kind { + fn of(value: &Value) -> Kind { + match value { + Value::Array(_) => Kind::Array, + Value::Object(_) => Kind::Object, + _ => Kind::Primitive, + } + } +} + +/// JSON primitive equality as JavaScript's `===` sees parsed JSON: numbers by +/// their double value, everything else exactly. +fn primitives_equal(left: &Value, right: &Value) -> bool { + match (left, right) { + (Value::Number(l), Value::Number(r)) => l.as_f64() == r.as_f64(), + _ => left == right, + } +} + +// Structural equality for a depth-capped pair, iterative so that a subtree as +// deep as the parser allows cannot exhaust the stack. +fn json_values_equal(cached: &Value, source: &Value) -> bool { + let mut pending = vec![(cached, source)]; + while let Some((left, right)) = pending.pop() { + match (left, right) { + (Value::Array(l), Value::Array(r)) => { + if l.len() != r.len() { + return false; + } + pending.extend(l.iter().zip(r)); + } + (Value::Object(l), Value::Object(r)) => { + if l.len() != r.len() { + return false; + } + for (name, value) in l { + let Some(other) = r.get(name) else { + return false; + }; + pending.push((value, other)); + } + } + _ => { + if Kind::of(left) != Kind::of(right) || !primitives_equal(left, right) { + return false; + } + } + } + } + true +} diff --git a/rust/src/protocol/mod.rs b/rust/src/protocol/mod.rs index 13c9c1bb..d7121248 100644 --- a/rust/src/protocol/mod.rs +++ b/rust/src/protocol/mod.rs @@ -1,8 +1,10 @@ //! The portable wire protocol: frames, envelopes, text conversion and -//! duration/timestamp domains (W04–W08 in `formal/guides/CONTRACTS.md`). +//! duration/timestamp domains (W04–W08 in `formal/guides/CONTRACTS.md`), +//! and the built-in confirmed-mismatch difference over JSON values. mod envelope; mod frame; +mod mismatch; mod text; pub use envelope::{ @@ -14,4 +16,6 @@ pub use frame::{ ceil_supported_cache_ttl_ms, decode_frame, encode_frame, normalize_read_result, parse_watermark, read_result_from_untrusted_json, validate_timestamp_ms, ProtocolError, }; +pub(crate) use mismatch::{bound_entry, diff_rendered}; +pub use mismatch::{diff_shadow_mismatch_json, MismatchJsonDiff}; pub use text::replacement_utf8; diff --git a/rust/src/shadow.rs b/rust/src/shadow.rs index 6c945742..b88a3e4c 100644 --- a/rust/src/shadow.rs +++ b/rust/src/shadow.rs @@ -17,10 +17,12 @@ use crate::flight::{start_pending, yield_deferred, Settled, ValueResult}; use crate::limits::DEFAULT_FALLBACK_TIMEOUT_MS; use crate::local::StoredValue; use crate::observe::{ - ErrorKind, Event, Layer, LogEvent, MissReason, ShadowMismatchDetails, ShadowOutcome, + ErrorKind, Event, Layer, LogEvent, MissReason, ShadowMismatchDetails, + ShadowMismatchLoggingOutcome, ShadowOutcome, }; -use crate::preview::{preview_key, preview_value}; +use crate::preview::preview_key; use crate::remote::Frame; +use crate::shadow_log::{render_mismatch_log, LogPlan, MismatchContent, Renderers}; /// One admitted shadow job holding an instance slot. /// One held entry of the instance-wide shadow table. Dropping it frees the @@ -93,6 +95,11 @@ struct Verdict { compared: Option<(StoredValue, StoredValue)>, } +/// A verdict's value age in seconds, clamped at zero. +fn age_seconds(age_ms: i64) -> f64 { + age_ms.max(0) as f64 / 1000.0 +} + impl Verdict { fn of(outcome: ShadowOutcome) -> Verdict { Verdict { @@ -184,44 +191,70 @@ impl Execution { self.emit(Event::ShadowValueAge { labels: self.labels.clone(), outcome: verdict.outcome, - seconds: age.max(0) as f64 / 1000.0, + seconds: age_seconds(age), }); } } + /// Emits the confirmed-mismatch warning and its one `ShadowMismatchLogging` + /// event. Runs after the verdict was recorded and never changes it; any + /// construction failure logs less, never more, and nothing reaches the + /// caller. async fn log_shadow_mismatch(&self, verdict: Verdict, slot: Arc) { - if verdict.outcome != ShadowOutcome::Mismatch || !self.policy.shadow.log_mismatches { + let shadow = self.policy.shadow; + let (ShadowOutcome::Mismatch, Some((cached, source)), Some(age_ms)) = + (verdict.outcome, verdict.compared, verdict.age_ms) + else { + return; + }; + if !shadow.logs_mismatches() { return; } - let (cached, source) = match (self.op.metadata.preview.clone(), verdict.compared) { - (Some(preview), Some((cached, source))) => { - let cpu_slot = slot.clone(); - crate::blocking::run(self.core.runtime.as_ref(), move || { - // Raw diagnostic work keeps capacity even if the awaiting - // async task is dropped during runtime shutdown. - let _slot = cpu_slot; - let render = |value: &StoredValue| { - catch_unwind(AssertUnwindSafe(|| preview(value))) - .ok() - .flatten() - .map(|json| preview_value(&json)) - }; - Ok((render(&cached), render(&source))) - }) - .await - .unwrap_or_default() - } - _ => (None, None), + let plan = LogPlan { + value: shadow.mismatch_value, + diff: shadow.mismatch_diff, + }; + let metadata = &self.op.metadata; + let renderers = Renderers { + value_renderer: metadata.value_renderer.clone(), + hook: metadata.mismatch_diff.clone(), }; - self.core + let content = if renderers.runs_code(plan) { + let cpu_slot = slot.clone(); + crate::blocking::run(self.core.runtime.as_ref(), move || { + // Raw diagnostic work keeps capacity even if the awaiting + // async task is dropped during runtime shutdown. + let _slot = cpu_slot; + Ok(render_mismatch_log(plan, &cached, &source, &renderers)) + }) + .await + // Rejected, panicked or dropped CPU work keeps the metadata warning. + .unwrap_or_else(|_| MismatchContent::failed()) + } else { + // Nothing to run: every requested field is unavailable. + render_mismatch_log(plan, &cached, &source, &renderers) + }; + let delivered = self + .core .log(LogEvent::ShadowMismatch(ShadowMismatchDetails { namespace: self.labels.namespace.clone(), use_case: self.labels.use_case.clone(), key_type: self.labels.key_type.clone(), + cached_value_age_seconds: age_seconds(age_ms), cache_key: preview_key(&self.keys.logical), - cached_value_json: cached, - source_value_json: source, + diff: content.diff, + values: content.values, + logging_outcome: content.failure, })); + let outcome = content.failure.unwrap_or(if delivered { + ShadowMismatchLoggingOutcome::Logged + } else { + ShadowMismatchLoggingOutcome::LoggerError + }); + self.emit(Event::ShadowMismatchLogging { + labels: self.labels.clone(), + outcome, + }); drop(slot); } diff --git a/rust/src/shadow_log.rs b/rust/src/shadow_log.rs new file mode 100644 index 00000000..74f6c034 --- /dev/null +++ b/rust/src/shadow_log.rs @@ -0,0 +1,408 @@ +//! Content of the confirmed-mismatch warning: the requested fields, each +//! failed closed, and the failure that explains them. + +use std::panic::{catch_unwind, AssertUnwindSafe}; + +use serde::Serialize; + +use crate::local::StoredValue; +use crate::observe::{ + ShadowMismatchDiffDetails, ShadowMismatchLoggingOutcome, ShadowMismatchValueDetails, +}; +use crate::operation::{ErasedMismatchDiff, ErasedPreview, MismatchEntry}; +use crate::policy::MismatchDiff; +use crate::preview::{ + clamp_utf8, preview_value, JsonPrefix, SHADOW_LOG_DIFF_MAX_BYTES, SHADOW_MISMATCH_MAX_ENTRIES, +}; +use crate::protocol::{bound_entry, diff_rendered, MismatchJsonDiff}; + +/// The effective content leaves of one confirmed-mismatch warning. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct LogPlan { + pub value: bool, + pub diff: MismatchDiff, +} + +/// One value's native JSON. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Rendering { + /// The value is absent: only the test-only marker, as Rust's default + /// codec stores no distinct absent value. + Absent, + Json(String), + /// Serialization failed or panicked. + Failed, +} + +/// The JSON rendering of an operation's values, captured where +/// `T: Serialize` is known. +#[derive(Clone, Copy)] +pub(crate) struct JsonRender { + render: fn(&StoredValue, bool) -> Rendering, +} + +impl JsonRender { + pub(crate) fn of() -> Self { + JsonRender { + render: render_json::, + } + } + + /// The complete rendering. + fn full(self, value: &StoredValue) -> Rendering { + (self.render)(value, false) + } + + /// The value field's prefix, streamed so value-only logging never holds + /// more than its cap; it checks the whole serialization all the same. + fn prefix(self, value: &StoredValue) -> Rendering { + (self.render)(value, true) + } +} + +fn render_json(value: &StoredValue, prefix: bool) -> Rendering { + catch_unwind(AssertUnwindSafe(|| { + let value = value + .downcast_ref::() + .expect("compared values have the operation's type"); + if prefix { + let mut json = JsonPrefix::new(); + match write_json(&mut json, value) { + Ok(true) => Rendering::Absent, + Ok(false) => Rendering::Json(json.finish()), + Err(_) => Rendering::Failed, + } + } else { + let mut json = Vec::with_capacity(128); + match write_json(&mut json, value) { + Ok(true) => Rendering::Absent, + Ok(false) => String::from_utf8(json).map_or(Rendering::Failed, Rendering::Json), + Err(_) => Rendering::Failed, + } + } + })) + .unwrap_or(Rendering::Failed) +} + +/// Serializes `value` as compact JSON; `Ok(true)`, with nothing written, when +/// the value is the absent marker. +#[cfg(not(feature = "test-util"))] +fn write_json(writer: W, value: &T) -> serde_json::Result { + value.serialize(&mut serde_json::Serializer::new(writer))?; + Ok(false) +} + +/// Serializes `value` as compact JSON; `Ok(true)`, with nothing written, when +/// the value is the absent marker. +#[cfg(feature = "test-util")] +fn write_json(writer: W, value: &T) -> serde_json::Result { + let mut absent = false; + value.serialize(absent::TopLevel { + json: &mut serde_json::Serializer::new(writer), + absent: &mut absent, + })?; + Ok(absent) +} + +/// Recognizes [`crate::testing::Absent`] at the top level of a value while +/// rendering it once: every call goes to the JSON serializer except a unit +/// struct carrying the marker's name. Nested markers render as `null`. +#[cfg(feature = "test-util")] +mod absent { + use serde::ser::{Serialize, Serializer}; + + type Json<'a, W> = &'a mut serde_json::Serializer; + type Result = std::result::Result; + + pub(super) struct TopLevel<'a, W> { + pub json: Json<'a, W>, + pub absent: &'a mut bool, + } + + macro_rules! forward { + ($($name:ident($($arg:ident: $ty:ty),*) -> $ret:ty;)*) => { + $(fn $name(self, $($arg: $ty),*) -> Result<$ret> { self.json.$name($($arg),*) })* + }; + } + + impl<'a, W: std::io::Write> Serializer for TopLevel<'a, W> { + type Ok = (); + type Error = serde_json::Error; + type SerializeSeq = as Serializer>::SerializeSeq; + type SerializeTuple = as Serializer>::SerializeTuple; + type SerializeTupleStruct = as Serializer>::SerializeTupleStruct; + type SerializeTupleVariant = as Serializer>::SerializeTupleVariant; + type SerializeMap = as Serializer>::SerializeMap; + type SerializeStruct = as Serializer>::SerializeStruct; + type SerializeStructVariant = as Serializer>::SerializeStructVariant; + + forward! { + serialize_bool(v: bool) -> (); + serialize_i8(v: i8) -> (); + serialize_i16(v: i16) -> (); + serialize_i32(v: i32) -> (); + serialize_i64(v: i64) -> (); + serialize_i128(v: i128) -> (); + serialize_u8(v: u8) -> (); + serialize_u16(v: u16) -> (); + serialize_u32(v: u32) -> (); + serialize_u64(v: u64) -> (); + serialize_u128(v: u128) -> (); + serialize_f32(v: f32) -> (); + serialize_f64(v: f64) -> (); + serialize_char(v: char) -> (); + serialize_str(v: &str) -> (); + serialize_bytes(v: &[u8]) -> (); + serialize_none() -> (); + serialize_unit() -> (); + serialize_unit_variant(name: &'static str, index: u32, variant: &'static str) -> (); + serialize_seq(len: Option) -> Self::SerializeSeq; + serialize_tuple(len: usize) -> Self::SerializeTuple; + serialize_tuple_struct(name: &'static str, len: usize) -> Self::SerializeTupleStruct; + serialize_tuple_variant(name: &'static str, index: u32, variant: &'static str, len: usize) -> Self::SerializeTupleVariant; + serialize_map(len: Option) -> Self::SerializeMap; + serialize_struct(name: &'static str, len: usize) -> Self::SerializeStruct; + serialize_struct_variant(name: &'static str, index: u32, variant: &'static str, len: usize) -> Self::SerializeStructVariant; + } + + fn serialize_unit_struct(self, name: &'static str) -> Result<()> { + if name == crate::testing::ABSENT_MARKER { + *self.absent = true; + return Ok(()); + } + self.json.serialize_unit_struct(name) + } + + fn serialize_some(self, value: &T) -> Result<()> { + self.json.serialize_some(value) + } + + fn serialize_newtype_struct( + self, + name: &'static str, + value: &T, + ) -> Result<()> { + self.json.serialize_newtype_struct(name, value) + } + + fn serialize_newtype_variant( + self, + name: &'static str, + index: u32, + variant: &'static str, + value: &T, + ) -> Result<()> { + self.json + .serialize_newtype_variant(name, index, variant, value) + } + } +} + +/// How a warning renders an operation's values: their JSON, which the value +/// fields log and the built-in walk reads, or a preview set in its place. A +/// preview replaces the JSON for the whole warning, so the walk never reads +/// what the value fields do not log, and difference fields then need the +/// hook. +#[derive(Clone)] +pub(crate) enum ValueRenderer { + Json(JsonRender), + Preview(ErasedPreview), +} + +/// What builds the fields: the operation's value renderer, if it has one, +/// and its difference hook. +#[derive(Clone, Default)] +pub(crate) struct Renderers { + pub value_renderer: Option, + pub hook: Option, +} + +impl Renderers { + /// The JSON rendering the warning derives from, if the values render as + /// JSON. + fn json(&self) -> Option { + match &self.value_renderer { + Some(ValueRenderer::Json(json)) => Some(*json), + _ => None, + } + } + + /// Whether building the plan's fields calls any renderer or hook, and so + /// belongs on the bounded CPU executor. + pub(crate) fn runs_code(&self, plan: LogPlan) -> bool { + let value = plan.value && self.value_renderer.is_some(); + let diff = plan.diff != MismatchDiff::Off && (self.hook.is_some() || self.json().is_some()); + value || diff + } +} + +/// The bounded content fields of one warning and the failure explaining any +/// unavailable one. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct MismatchContent { + pub diff: Option, + pub values: Option, + pub failure: Option, +} + +impl MismatchContent { + /// The metadata-only warning after construction itself failed. + pub(crate) fn failed() -> Self { + MismatchContent { + diff: None, + values: None, + failure: Some(ShadowMismatchLoggingOutcome::RenderError), + } + } +} + +/// Builds the content fields the plan requests for one confirmed mismatch. +/// +/// Each side is rendered to JSON at most once, and every built-in field +/// derives from that rendering. Every failure fails its fields closed; a +/// failed hook never falls back to the built-in computation. An absent side +/// is not a failure: its value field is unavailable, and the built-in +/// reports the other side whole at the root. The hook replaces the walk, so +/// with `value` off and a hook neither side is rendered. A preview replaces +/// the JSON rendering, so with a preview the difference fields come only +/// from the hook. +pub(crate) fn render_mismatch_log( + plan: LogPlan, + cached: &StoredValue, + source: &StoredValue, + renderers: &Renderers, +) -> MismatchContent { + let json = renderers.json(); + let walk = plan.diff != MismatchDiff::Off && renderers.hook.is_none(); + let rendering = |value: &StoredValue| -> Option { + let json = json?; + if walk { + Some(json.full(value)) + } else if plan.value { + Some(json.prefix(value)) + } else { + None + } + }; + let (cached_json, source_json) = (rendering(cached), rendering(source)); + let mut render_failed = false; + let mut diff_failed = false; + + let values = plan.value.then(|| { + let mut field = |value: &StoredValue, json: &Option| { + let text = match (&renderers.value_renderer, json) { + (_, Some(Rendering::Absent)) => return None, + (Some(ValueRenderer::Preview(preview)), _) => { + match catch_unwind(AssertUnwindSafe(|| preview(value))) { + Ok(Some(text)) => Some(text), + _ => None, + } + } + (_, Some(Rendering::Json(json))) => Some(json.clone()), + _ => None, + }; + render_failed |= text.is_none(); + text.map(|text| preview_value(&text)) + }; + ShadowMismatchValueDetails { + cached_value_json: field(cached, &cached_json), + source_value_json: field(source, &source_json), + } + }); + + let diff = (plan.diff != MismatchDiff::Off).then(|| { + let with_values = plan.diff == MismatchDiff::Values; + let computed = if let Some(hook) = &renderers.hook { + let entries = run_hook(hook, cached, source, with_values); + diff_failed = entries.is_none(); + entries + } else { + match (rendered_side(&cached_json), rendered_side(&source_json)) { + (Some(cached), Some(source)) => match diff_rendered(cached, source, with_values) { + Ok(diff) => Some(diff), + // A rendering too deep for the parser fails the walk. + Err(_) => { + diff_failed = true; + None + } + }, + // A side without a rendering, or a warning without JSON: an + // operation built with a codec (lan17/DialCache#245 tracks a + // renderer for those), or one whose preview replaces its JSON + // or was set to `None`. + _ => { + render_failed = true; + None + } + } + }; + match computed { + None => ShadowMismatchDiffDetails { + level: plan.diff, + mismatch_paths: None, + mismatch_path_count: None, + diff_json: None, + }, + Some(diff) => ShadowMismatchDiffDetails { + level: plan.diff, + diff_json: with_values.then(|| render_diff_json(&diff.entries)), + mismatch_paths: Some(diff.entries.into_iter().map(|entry| entry.path).collect()), + mismatch_path_count: Some(diff.count), + }, + } + }); + + MismatchContent { + diff, + values, + failure: if diff_failed { + Some(ShadowMismatchLoggingOutcome::HookError) + } else if render_failed { + Some(ShadowMismatchLoggingOutcome::RenderError) + } else { + None + }, + } +} + +/// A side the walk can compare: `Some(None)` when absent, `None` when it has +/// no rendering. +fn rendered_side(json: &Option) -> Option> { + match json { + Some(Rendering::Json(text)) => Some(Some(text.as_str())), + Some(Rendering::Absent) => Some(None), + Some(Rendering::Failed) | None => None, + } +} + +/// Runs the hook under the comparator's discipline and bounds its entries; +/// `None` when it panicked. +fn run_hook( + hook: &ErasedMismatchDiff, + cached: &StoredValue, + source: &StoredValue, + with_values: bool, +) -> Option { + let entries = catch_unwind(AssertUnwindSafe(|| hook(cached, source))).ok()?; + Some(MismatchJsonDiff { + count: entries.len() as u64, + entries: entries + .iter() + .take(SHADOW_MISMATCH_MAX_ENTRIES) + .map(|entry| { + bound_entry( + &entry.path, + entry.cached.as_ref(), + entry.source.as_ref(), + with_values, + ) + }) + .collect(), + }) +} + +/// The bounded entries as one JSON array, clamped to the field's cap. +fn render_diff_json(entries: &[MismatchEntry]) -> String { + let json = serde_json::to_string(entries).expect("JSON values serialize"); + clamp_utf8(&json, SHADOW_LOG_DIFF_MAX_BYTES) +} diff --git a/rust/src/shadow_log_tests.rs b/rust/src/shadow_log_tests.rs new file mode 100644 index 00000000..41d447e3 --- /dev/null +++ b/rust/src/shadow_log_tests.rs @@ -0,0 +1,938 @@ +//! Native tests of the built-in mismatch walk and of warning content: every +//! cap, absent sides, failures and the hook. The W10 vectors run from +//! `formal/catalogs/protocol-vectors.json` through +//! `tests/formal/mismatch_vectors.rs`. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use serde::ser::Error as _; +use serde::{Serialize, Serializer}; +use serde_json::{json, Value}; + +use crate::local::StoredValue; +use crate::observe::{ + ShadowMismatchDiffDetails, ShadowMismatchLoggingOutcome, ShadowMismatchValueDetails, +}; +use crate::operation::{ErasedMismatchDiff, ErasedPreview, MismatchEntry}; +use crate::policy::MismatchDiff; +use crate::preview::{ + SHADOW_LOG_DIFF_MAX_BYTES, SHADOW_LOG_TRUNCATION_MARKER, SHADOW_LOG_VALUE_MAX_BYTES, +}; +use crate::protocol::diff_shadow_mismatch_json; +use crate::shadow_log::{ + render_mismatch_log, JsonRender, LogPlan, MismatchContent, Renderers, ValueRenderer, +}; + +const VALUE_ONLY: LogPlan = LogPlan { + value: true, + diff: MismatchDiff::Off, +}; +const PATHS_ONLY: LogPlan = LogPlan { + value: false, + diff: MismatchDiff::Paths, +}; +const VALUES_ONLY: LogPlan = LogPlan { + value: false, + diff: MismatchDiff::Values, +}; +const EVERYTHING: LogPlan = LogPlan { + value: true, + diff: MismatchDiff::Values, +}; + +const HOOK_ERROR: Option = + Some(ShadowMismatchLoggingOutcome::HookError); +const RENDER_ERROR: Option = + Some(ShadowMismatchLoggingOutcome::RenderError); + +const CACHED_USER_JSON: &str = r#"{"id":"123","email":"old@example.com"}"#; +const SOURCE_USER_JSON: &str = r#"{"id":"123","email":"new@example.com"}"#; + +/// A compared value: JSON, a record serialized in field order, the test-only +/// absent marker, a value whose serialization fails or panics, or JSON that +/// counts its serializations. +#[derive(Clone)] +enum V { + Json(Value), + User(User), + Absent, + Fails, + Panics, + Counted(Value, Arc), +} + +#[derive(Clone, Serialize)] +struct User { + id: &'static str, + email: &'static str, +} + +impl Serialize for V { + fn serialize(&self, serializer: S) -> Result { + match self { + V::Json(value) => value.serialize(serializer), + V::User(user) => user.serialize(serializer), + V::Absent => crate::testing::Absent.serialize(serializer), + V::Fails => Err(S::Error::custom("not serializable")), + V::Panics => panic!("serialization panicked"), + V::Counted(value, renders) => { + renders.fetch_add(1, Ordering::SeqCst); + value.serialize(serializer) + } + } + } +} + +fn cached_user() -> V { + V::User(User { + id: "123", + email: "old@example.com", + }) +} + +fn source_user() -> V { + V::User(User { + id: "123", + email: "new@example.com", + }) +} + +fn stored(value: V) -> StoredValue { + Arc::new(value) +} + +fn json_renderers() -> Renderers { + Renderers { + value_renderer: Some(ValueRenderer::Json(JsonRender::of::())), + ..Renderers::default() + } +} + +fn previewed(preview: ErasedPreview) -> Renderers { + Renderers { + value_renderer: Some(ValueRenderer::Preview(preview)), + ..Renderers::default() + } +} + +fn with_hook(hook: impl Fn(&V, &V) -> Vec + Send + Sync + 'static) -> Renderers { + let hook: ErasedMismatchDiff = Arc::new(move |cached: &StoredValue, source: &StoredValue| { + hook( + cached.downcast_ref().unwrap(), + source.downcast_ref().unwrap(), + ) + }); + Renderers { + hook: Some(hook), + ..json_renderers() + } +} + +fn render(plan: LogPlan, cached: V, source: V) -> MismatchContent { + render_with(plan, cached, source, &json_renderers()) +} + +fn render_with(plan: LogPlan, cached: V, source: V, renderers: &Renderers) -> MismatchContent { + render_mismatch_log(plan, &stored(cached), &stored(source), renderers) +} + +fn diff_fields( + level: MismatchDiff, + paths: Option<&[&str]>, + count: Option, + diff_json: Option<&str>, +) -> Option { + Some(ShadowMismatchDiffDetails { + level, + mismatch_paths: paths.map(|paths| paths.iter().map(|path| path.to_string()).collect()), + mismatch_path_count: count, + diff_json: diff_json.map(str::to_owned), + }) +} + +fn values(cached: Option<&str>, source: Option<&str>) -> Option { + Some(ShadowMismatchValueDetails { + cached_value_json: cached.map(str::to_owned), + source_value_json: source.map(str::to_owned), + }) +} + +fn content( + diff: Option, + values: Option, + failure: Option, +) -> MismatchContent { + MismatchContent { + diff, + values, + failure, + } +} + +/// The diff fields after the hook or the walk failed, at `plan`'s level. +fn failed_diff(plan: LogPlan) -> Option { + diff_fields(plan.diff, None, None, None) +} + +fn entries_json(diff: &crate::protocol::MismatchJsonDiff) -> Value { + serde_json::to_value(&diff.entries).unwrap() +} + +fn nest_members(depth: usize, leaf: Value) -> Value { + (0..depth).fold(leaf, |value, _| json!({ "n": value })) +} + +#[test] +fn walk_reads_members_named_like_object_intrinsics_as_ordinary_members() { + let cached: Value = serde_json::from_str(r#"{"constructor":1,"__proto__":{"a":1}}"#).unwrap(); + let source: Value = + serde_json::from_str(r#"{"toString":"x","valueOf":null,"__proto__":{"a":2}}"#).unwrap(); + let entries = json!([ + {"path": "/__proto__/a", "cached": 1, "source": 2}, + {"path": "/constructor", "cached": 1}, + {"path": "/toString", "source": "x"}, + {"path": "/valueOf", "source": null}, + ]); + let diff = diff_shadow_mismatch_json(&cached, &source); + assert_eq!((entries_json(&diff), diff.count), (entries.clone(), 4)); + let fields = render(VALUES_ONLY, V::Json(cached), V::Json(source)) + .diff + .unwrap(); + let logged: Value = serde_json::from_str(fields.diff_json.as_deref().unwrap()).unwrap(); + assert_eq!(logged, entries); +} + +#[test] +fn a_depth_capped_pair_compares_only_its_own_members() { + let nest = |leaf: &str| -> Value { + serde_json::from_str(&format!( + "{}{leaf}{}", + r#"{"a":"#.repeat(32), + "}".repeat(32) + )) + .unwrap() + }; + let diff = diff_shadow_mismatch_json(&nest(r#"{"__proto__":{}}"#), &nest(r#"{"b":{}}"#)); + assert_eq!( + (entries_json(&diff), diff.count), + ( + json!([{"path": "/a".repeat(32), "cached": {"__proto__": {}}, "source": {"b": {}}}]), + 1 + ) + ); +} + +#[test] +fn the_walk_cuts_entries_at_64_in_walk_order_and_counts_every_difference() { + let (mut cached, mut source) = (serde_json::Map::new(), serde_json::Map::new()); + for index in (0..70).rev() { + cached.insert(format!("m{index:02}"), json!(0)); + source.insert(format!("m{index:02}"), json!(1)); + } + let entries: Vec = (0..64) + .map(|index| json!({"path": format!("/m{index:02}"), "cached": 0, "source": 1})) + .collect(); + let (cached, source) = (Value::Object(cached), Value::Object(source)); + let diff = diff_shadow_mismatch_json(&cached, &source); + assert_eq!( + (entries_json(&diff), diff.count), + (Value::Array(entries.clone()), 70) + ); + let paths: Vec<&str> = entries + .iter() + .map(|entry| entry["path"].as_str().unwrap()) + .collect(); + assert_eq!( + render(PATHS_ONLY, V::Json(cached), V::Json(source)), + content( + diff_fields(MismatchDiff::Paths, Some(&paths), Some(70), None), + None, + None + ) + ); +} + +#[test] +fn each_pointer_is_clamped_to_512_bytes_on_a_character_boundary() { + let names = ["k".repeat(511), "k".repeat(600), "é".repeat(300)]; + let cached = Value::Object(names.iter().map(|name| (name.clone(), json!(1))).collect()); + let source = Value::Object(names.iter().map(|name| (name.clone(), json!(2))).collect()); + let diff = diff_shadow_mismatch_json(&cached, &source); + assert_eq!( + (entries_json(&diff), diff.count), + ( + json!([ + {"path": format!("/{}", "k".repeat(511)), "cached": 1, "source": 2}, + {"path": format!("/{}{SHADOW_LOG_TRUNCATION_MARKER}", "k".repeat(497)), "cached": 1, "source": 2}, + {"path": format!("/{}{SHADOW_LOG_TRUNCATION_MARKER}", "é".repeat(248)), "cached": 1, "source": 2}, + ]), + 3 + ) + ); +} + +#[test] +fn each_side_over_1_kib_becomes_its_clamped_json_text() { + let list = vec![1000; 300]; + let cached = json!({ + "accent": "é".repeat(600), + "blob": "x".repeat(2000), + "edge": "e".repeat(1022), + "extra": "z".repeat(1100), + "list": list, + }); + let source = json!({"accent": "a", "blob": "y", "edge": "f".repeat(1022), "list": [1000]}); + let list_json = serde_json::to_string(&list).unwrap(); + let diff = diff_shadow_mismatch_json(&cached, &source); + assert_eq!( + (entries_json(&diff), diff.count), + ( + json!([ + {"path": "/accent", "cached": format!("\"{}{SHADOW_LOG_TRUNCATION_MARKER}", "é".repeat(504)), "source": "a"}, + {"path": "/blob", "cached": format!("\"{}{SHADOW_LOG_TRUNCATION_MARKER}", "x".repeat(1009)), "source": "y"}, + {"path": "/edge", "cached": "e".repeat(1022), "source": "f".repeat(1022)}, + {"path": "/extra", "cached": format!("\"{}{SHADOW_LOG_TRUNCATION_MARKER}", "z".repeat(1009))}, + {"path": "/list", "cached": format!("{}{SHADOW_LOG_TRUNCATION_MARKER}", &list_json[..1010]), "source": [1000]}, + ]), + 5 + ) + ); +} + +#[test] +fn the_walk_stops_at_32_reference_tokens_however_deep_the_difference_lies() { + let diff = diff_shadow_mismatch_json(&nest_members(40, json!(1)), &nest_members(40, json!(2))); + assert_eq!( + (entries_json(&diff), diff.count), + ( + json!([{"path": "/n".repeat(32), "cached": nest_members(8, json!(1)), "source": nest_members(8, json!(2))}]), + 1 + ) + ); + let clamped = format!("{}{SHADOW_LOG_TRUNCATION_MARKER}", r#"{"n":"#.repeat(202)); + let diff = + diff_shadow_mismatch_json(&nest_members(1000, json!(1)), &nest_members(1000, json!(2))); + assert_eq!( + (entries_json(&diff), diff.count), + ( + json!([{"path": "/n".repeat(32), "cached": clamped, "source": clamped}]), + 1 + ) + ); +} + +#[test] +fn equal_containers_past_the_depth_cap_compare_equal_regardless_of_member_order() { + let (mut cached, mut source) = (json!("leaf"), json!("leaf")); + for level in 0..1000 { + cached = json!({"a": level, "next": cached}); + source = json!({"next": source, "a": level}); + } + let diff = diff_shadow_mismatch_json(&cached, &source); + assert_eq!((entries_json(&diff), diff.count), (json!([]), 0)); +} + +#[test] +fn numbers_compare_by_value_as_parsed_json_does() { + let diff = diff_shadow_mismatch_json( + &json!({"a": 1, "b": -0.0, "c": 1.5}), + &json!({"a": 1.0, "b": 0, "c": 1.25}), + ); + assert_eq!( + (entries_json(&diff), diff.count), + (json!([{"path": "/c", "cached": 1.5, "source": 1.25}]), 1) + ); +} + +#[test] +fn renders_exactly_the_fields_each_plan_requests() { + let diff_json = r#"[{"path":"/email","cached":"old@example.com","source":"new@example.com"}]"#; + let user_values = values(Some(CACHED_USER_JSON), Some(SOURCE_USER_JSON)); + for (plan, expected) in [ + ( + PATHS_ONLY, + content( + diff_fields(MismatchDiff::Paths, Some(&["/email"]), Some(1), None), + None, + None, + ), + ), + ( + VALUES_ONLY, + content( + diff_fields( + MismatchDiff::Values, + Some(&["/email"]), + Some(1), + Some(diff_json), + ), + None, + None, + ), + ), + (VALUE_ONLY, content(None, user_values.clone(), None)), + ( + LogPlan { + value: true, + diff: MismatchDiff::Paths, + }, + content( + diff_fields(MismatchDiff::Paths, Some(&["/email"]), Some(1), None), + user_values.clone(), + None, + ), + ), + ( + EVERYTHING, + content( + diff_fields( + MismatchDiff::Values, + Some(&["/email"]), + Some(1), + Some(diff_json), + ), + user_values.clone(), + None, + ), + ), + ] { + assert_eq!( + render(plan, cached_user(), source_user()), + expected, + "{plan:?}" + ); + } +} + +#[test] +fn byte_identical_renderings_have_no_entries() { + // The comparator saw a difference the JSON does not show. + assert_eq!( + render( + EVERYTHING, + V::Json(json!({"id": 1})), + V::Json(json!({"id": 1})) + ), + content( + diff_fields(MismatchDiff::Values, Some(&[]), Some(0), Some("[]")), + values(Some(r#"{"id":1}"#), Some(r#"{"id":1}"#)), + None + ) + ); +} + +#[test] +fn byte_clamps_value_fields_without_splitting_utf8_sequences() { + let value = V::Json(json!({"text": "🙂".repeat(SHADOW_LOG_VALUE_MAX_BYTES)})); + let fields = render(VALUE_ONLY, value.clone(), value).values.unwrap(); + let cached = fields.cached_value_json.unwrap(); + assert!(cached.len() <= SHADOW_LOG_VALUE_MAX_BYTES); + assert!(cached.ends_with(SHADOW_LOG_TRUNCATION_MARKER)); + assert_eq!(fields.source_value_json, Some(cached)); +} + +#[test] +fn an_absent_root_is_one_side_at_the_root_pointer_not_a_render_failure() { + assert_eq!( + render(EVERYTHING, V::Absent, V::Json(json!({"version": 2}))), + content( + diff_fields( + MismatchDiff::Values, + Some(&[""]), + Some(1), + Some(r#"[{"path":"","source":{"version":2}}]"#) + ), + values(None, Some(r#"{"version":2}"#)), + None + ) + ); + assert_eq!( + render(EVERYTHING, V::Json(json!({"version": 2})), V::Absent), + content( + diff_fields( + MismatchDiff::Values, + Some(&[""]), + Some(1), + Some(r#"[{"path":"","cached":{"version":2}}]"#) + ), + values(Some(r#"{"version":2}"#), None), + None + ) + ); + assert_eq!( + render(PATHS_ONLY, V::Absent, V::Json(Value::Null)), + content( + diff_fields(MismatchDiff::Paths, Some(&[""]), Some(1), None), + None, + None + ) + ); + // A stored null is a value, so it stays distinct from absence. + assert_eq!( + render(VALUES_ONLY, V::Absent, V::Json(Value::Null)) + .diff + .unwrap() + .diff_json + .as_deref(), + Some(r#"[{"path":"","source":null}]"#) + ); + // Absence beside a side with no rendering is still a render failure. + assert_eq!( + render(EVERYTHING, V::Absent, V::Fails), + content(failed_diff(EVERYTHING), values(None, None), RENDER_ERROR) + ); + // A custom comparator may call two absent values different; no field differs. + assert_eq!( + render(EVERYTHING, V::Absent, V::Absent), + content( + diff_fields(MismatchDiff::Values, Some(&[]), Some(0), Some("[]")), + values(None, None), + None + ) + ); + // The value-only prefix rendering recognizes the marker too. + assert_eq!( + render(VALUE_ONLY, V::Absent, source_user()), + content(None, values(None, Some(SOURCE_USER_JSON)), None) + ); +} + +#[test] +fn a_root_without_a_rendering_fails_the_fields_it_feeds_with_render_error() { + let version = || V::Json(json!({"version": 2})); + for unrenderable in [V::Fails, V::Panics] { + assert_eq!( + render(EVERYTHING, unrenderable.clone(), version()), + content( + failed_diff(EVERYTHING), + values(None, Some(r#"{"version":2}"#)), + RENDER_ERROR + ) + ); + assert_eq!( + render(EVERYTHING, version(), unrenderable.clone()), + content( + failed_diff(EVERYTHING), + values(Some(r#"{"version":2}"#), None), + RENDER_ERROR + ) + ); + assert_eq!( + render(PATHS_ONLY, version(), unrenderable.clone()), + content(failed_diff(PATHS_ONLY), None, RENDER_ERROR) + ); + assert_eq!( + render(VALUE_ONLY, unrenderable, version()), + content(None, values(None, Some(r#"{"version":2}"#)), RENDER_ERROR) + ); + } +} + +#[test] +fn each_side_is_serialized_once_for_every_field_it_feeds() { + let (cached_renders, source_renders) = + (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + assert_eq!( + render( + EVERYTHING, + V::Counted(json!({"email": "a"}), cached_renders.clone()), + V::Counted(json!({"email": "b"}), source_renders.clone()), + ), + content( + diff_fields( + MismatchDiff::Values, + Some(&["/email"]), + Some(1), + Some(r#"[{"path":"/email","cached":"a","source":"b"}]"#) + ), + values(Some(r#"{"email":"a"}"#), Some(r#"{"email":"b"}"#)), + None + ) + ); + assert_eq!(cached_renders.load(Ordering::SeqCst), 1); + assert_eq!(source_renders.load(Ordering::SeqCst), 1); +} + +#[test] +fn diff_json_is_clamped_to_8_kib_with_the_truncation_marker() { + let (mut cached, mut source, mut entries) = + (serde_json::Map::new(), serde_json::Map::new(), Vec::new()); + for index in 0..20 { + let name = format!("m{index:02}"); + cached.insert(name.clone(), json!("x".repeat(500))); + source.insert(name.clone(), json!("y".repeat(500))); + entries.push(MismatchEntry { + path: format!("/{name}"), + cached: Some(json!("x".repeat(500))), + source: Some(json!("y".repeat(500))), + }); + } + // The 8 KiB bound is the record's, so it is written out rather than taken + // from the implementation's constant. + let whole = serde_json::to_string(&entries).unwrap(); + let diff_json = format!( + "{}{SHADOW_LOG_TRUNCATION_MARKER}", + &whole[..8192 - SHADOW_LOG_TRUNCATION_MARKER.len()] + ); + assert_eq!(diff_json.len(), 8192); + assert_eq!(SHADOW_LOG_DIFF_MAX_BYTES, 8192); + let paths: Vec<&str> = entries.iter().map(|entry| entry.path.as_str()).collect(); + assert_eq!( + render( + VALUES_ONLY, + V::Json(Value::Object(cached)), + V::Json(Value::Object(source)) + ), + content( + diff_fields( + MismatchDiff::Values, + Some(&paths), + Some(20), + Some(&diff_json) + ), + None, + None + ) + ); +} + +#[test] +fn a_rendering_too_deep_to_parse_fails_the_walk_closed_with_hook_error() { + // serde_json parses at most 128 nested levels; the walk never falls back. + let deep = || V::Json(nest_members(200, json!(1))); + let deep_json = serde_json::to_string(&nest_members(200, json!(1))).unwrap(); + let content_of = render(EVERYTHING, deep(), V::Json(json!(1))); + assert_eq!(content_of.diff, failed_diff(EVERYTHING)); + assert_eq!(content_of.failure, HOOK_ERROR); + assert_eq!( + content_of.values.unwrap().cached_value_json, + Some(crate::preview::preview_value(&deep_json)) + ); + assert_eq!( + render(PATHS_ONLY, V::Json(json!(1)), deep()), + content(failed_diff(PATHS_ONLY), None, HOOK_ERROR) + ); +} + +#[test] +fn the_hook_supplies_the_entries_and_the_walk_never_runs() { + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + let renderers = with_hook(move |cached, source| { + seen.fetch_add(1, Ordering::SeqCst); + assert!( + matches!((cached, source), (V::Json(c), V::Json(s)) if c["id"] == 1 && s["id"] == 2) + ); + vec![ + MismatchEntry { + path: "/profile/email".into(), + cached: Some(json!("old@example.com")), + source: Some(json!("new@example.com")), + }, + MismatchEntry { + path: "/roles".into(), + cached: None, + source: None, + }, + ] + }); + assert_eq!( + render_with( + EVERYTHING, + V::Json(json!({"id": 1})), + V::Json(json!({"id": 2})), + &renderers + ), + content( + diff_fields( + MismatchDiff::Values, + Some(&["/profile/email", "/roles"]), + Some(2), + Some( + r#"[{"path":"/profile/email","cached":"old@example.com","source":"new@example.com"},{"path":"/roles"}]"# + ) + ), + values(Some(r#"{"id":1}"#), Some(r#"{"id":2}"#)), + None + ) + ); + assert_eq!(calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn neither_root_side_is_rendered_when_the_hook_supplies_the_entries_and_value_is_off() { + let (cached_renders, source_renders) = + (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + assert_eq!( + render_with( + VALUES_ONLY, + V::Counted(json!("cached"), cached_renders.clone()), + V::Counted(json!("source"), source_renders.clone()), + &with_hook(|_, _| Vec::new()), + ), + content( + diff_fields(MismatchDiff::Values, Some(&[]), Some(0), Some("[]")), + None, + None + ) + ); + assert_eq!(cached_renders.load(Ordering::SeqCst), 0); + assert_eq!(source_renders.load(Ordering::SeqCst), 0); + // Hook paths without values need no rendering either, even of a value that has none. + assert_eq!( + render_with( + PATHS_ONLY, + cached_user(), + V::Fails, + &with_hook(|_, _| vec![MismatchEntry { + path: "/version".into(), + cached: None, + source: None + }]) + ), + content( + diff_fields(MismatchDiff::Paths, Some(&["/version"]), Some(1), None), + None, + None + ) + ); +} + +#[test] +fn a_panicking_hook_fails_closed_with_hook_error_and_no_fallback() { + let renderers = with_hook(|_, _| panic!("hook failed")); + let user_values = values(Some(CACHED_USER_JSON), Some(SOURCE_USER_JSON)); + assert_eq!( + render_with(EVERYTHING, cached_user(), source_user(), &renderers), + content(failed_diff(EVERYTHING), user_values, HOOK_ERROR) + ); + assert_eq!( + render_with(PATHS_ONLY, cached_user(), source_user(), &renderers), + content(failed_diff(PATHS_ONLY), None, HOOK_ERROR) + ); +} + +#[test] +fn the_hook_is_not_called_when_diff_is_off() { + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + let renderers = with_hook(move |_, _| { + seen.fetch_add(1, Ordering::SeqCst); + Vec::new() + }); + assert_eq!( + render_with(VALUE_ONLY, cached_user(), source_user(), &renderers), + content( + None, + values(Some(CACHED_USER_JSON), Some(SOURCE_USER_JSON)), + None + ) + ); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn hook_entries_omit_absent_sides_and_are_clamped_and_cut_at_64() { + let long_path = format!("/{}", "k".repeat(600)); + let clamped_path = format!("/{}{SHADOW_LOG_TRUNCATION_MARKER}", "k".repeat(497)); + let hook_path = long_path.clone(); + let renderers = with_hook(move |_, _| { + let mut entries = vec![MismatchEntry { + path: hook_path.clone(), + cached: Some(json!("x".repeat(2000))), + source: Some(json!({"ok": true})), + }]; + entries.extend((1..70).map(|index| MismatchEntry { + path: format!("/{index}"), + cached: None, + source: None, + })); + entries + }); + let fields = render_with(VALUES_ONLY, cached_user(), source_user(), &renderers); + assert_eq!(fields.failure, None); + let fields = fields.diff.unwrap(); + let retained: Vec = (1..64).map(|index| format!("/{index}")).collect(); + let mut paths = vec![clamped_path.clone()]; + paths.extend(retained.iter().cloned()); + assert_eq!(fields.mismatch_paths, Some(paths)); + assert_eq!(fields.mismatch_path_count, Some(70)); + let mut expected = vec![json!({ + "path": clamped_path, + "cached": format!("\"{}{SHADOW_LOG_TRUNCATION_MARKER}", "x".repeat(1009)), + "source": {"ok": true}, + })]; + expected.extend(retained.iter().map(|path| json!({"path": path}))); + let logged: Value = serde_json::from_str(fields.diff_json.as_deref().unwrap()).unwrap(); + assert_eq!(logged, Value::Array(expected)); + + let renderers = with_hook(|_, _| { + vec![ + MismatchEntry { + path: "/a".into(), + cached: None, + source: None, + }, + MismatchEntry { + path: "/b".into(), + cached: Some(json!(1)), + source: None, + }, + MismatchEntry { + path: "/c".into(), + cached: None, + source: Some(Value::Null), + }, + ] + }); + assert_eq!( + render_with(VALUES_ONLY, cached_user(), source_user(), &renderers) + .diff + .unwrap() + .diff_json + .as_deref(), + Some(r#"[{"path":"/a"},{"path":"/b","cached":1},{"path":"/c","source":null}]"#) + ); +} + +#[test] +fn hook_entries_stay_when_a_root_side_has_no_rendering() { + let renderers = with_hook(|_, _| { + vec![MismatchEntry { + path: "/a".into(), + cached: None, + source: None, + }] + }); + assert_eq!( + render_with(EVERYTHING, V::Fails, source_user(), &renderers), + content( + diff_fields( + MismatchDiff::Values, + Some(&["/a"]), + Some(1), + Some(r#"[{"path":"/a"}]"#) + ), + values(None, Some(SOURCE_USER_JSON)), + RENDER_ERROR + ) + ); +} + +#[test] +fn hook_error_ranks_above_render_error() { + let renderers = with_hook(|_, _| panic!("hook failed")); + assert_eq!( + render_with(EVERYTHING, V::Fails, source_user(), &renderers), + content( + failed_diff(EVERYTHING), + values(None, Some(SOURCE_USER_JSON)), + HOOK_ERROR + ) + ); +} + +#[test] +fn an_operation_without_json_needs_a_preview_and_a_hook() { + let no_json = Renderers::default(); + assert_eq!( + render_with(EVERYTHING, cached_user(), source_user(), &no_json), + content(failed_diff(EVERYTHING), values(None, None), RENDER_ERROR) + ); + assert!(!no_json.runs_code(EVERYTHING)); + let preview: ErasedPreview = Arc::new(|value: &StoredValue| match value.downcast_ref::() { + Some(V::User(user)) => Some(format!("preview {:?}", user.email)), + _ => None, + }); + let hook: ErasedMismatchDiff = Arc::new(|_: &StoredValue, _: &StoredValue| { + vec![MismatchEntry { + path: "/email".into(), + cached: None, + source: None, + }] + }); + let custom = Renderers { + hook: Some(hook), + ..previewed(preview) + }; + assert!(custom.runs_code(PATHS_ONLY)); + assert_eq!( + render_with(EVERYTHING, cached_user(), source_user(), &custom), + content( + diff_fields( + MismatchDiff::Values, + Some(&["/email"]), + Some(1), + Some(r#"[{"path":"/email"}]"#) + ), + values( + Some(r#"preview "old@example.com""#), + Some(r#"preview "new@example.com""#) + ), + None + ) + ); +} + +#[test] +fn a_preview_replaces_the_json_rendering_for_the_whole_warning() { + // An id-only preview redacts the email, so the walk must not log it. + let redacting: ErasedPreview = + Arc::new(|value: &StoredValue| match value.downcast_ref::() { + Some(V::User(user)) => Some(format!(r#"{{"id":"{}"}}"#, user.id)), + _ => None, + }); + let redacted = values(Some(r#"{"id":"123"}"#), Some(r#"{"id":"123"}"#)); + let renderers = previewed(redacting.clone()); + assert!(!renderers.runs_code(PATHS_ONLY)); + assert_eq!( + render_with(EVERYTHING, cached_user(), source_user(), &renderers), + content(failed_diff(EVERYTHING), redacted.clone(), RENDER_ERROR) + ); + // With a hook, the difference fields come from the hook alone. + let hooked = Renderers { + value_renderer: Some(ValueRenderer::Preview(redacting)), + ..with_hook(|_, _| { + vec![MismatchEntry { + path: "/email".into(), + cached: None, + source: None, + }] + }) + }; + assert_eq!( + render_with(EVERYTHING, cached_user(), source_user(), &hooked), + content( + diff_fields( + MismatchDiff::Values, + Some(&["/email"]), + Some(1), + Some(r#"[{"path":"/email"}]"#) + ), + redacted, + None + ) + ); +} + +#[test] +fn a_preview_fails_closed_per_value_and_receives_the_absent_marker() { + let preview: ErasedPreview = Arc::new(|value: &StoredValue| match value.downcast_ref::() { + Some(V::User(user)) if user.email == "new@example.com" => Some("🙂".repeat(10_000)), + Some(V::User(_)) => None, + _ => panic!("preview panicked"), + }); + let renderers = previewed(preview); + let source_field = crate::preview::preview_value(&"🙂".repeat(10_000)); + assert_eq!( + render_with(VALUE_ONLY, cached_user(), source_user(), &renderers), + content(None, values(None, Some(&source_field)), RENDER_ERROR) + ); + // No JSON rendering runs, so the preview, which panics here, receives the + // test-only absent marker. + assert_eq!( + render_with(VALUE_ONLY, V::Absent, source_user(), &renderers), + content(None, values(None, Some(&source_field)), RENDER_ERROR) + ); +} diff --git a/rust/src/testing.rs b/rust/src/testing.rs index 2f18cc35..501f5499 100644 --- a/rust/src/testing.rs +++ b/rust/src/testing.rs @@ -23,6 +23,27 @@ use parking_lot::Mutex; /// Epoch of every controlled history: 2026-09-08T12:00:00.000Z. pub const WALL_EPOCH_MS: i64 = 1_788_868_800_000; +/// The name [`Absent`] serializes as a unit struct. +pub const ABSENT_MARKER: &str = "$dialcache::testing::Absent"; + +/// The test-only absent value of confirmed-mismatch warnings. +/// +/// Rust's default codec stores no distinct absent value, unlike TypeScript's +/// top-level `undefined`. A test value type whose `Serialize` delegates to +/// this marker at the top level stands for one: a warning treats it as an +/// absent side, not as a value that failed to render, so its value field is +/// unavailable without a `loggingOutcome` and the built-in difference +/// reports the other side whole at the root pointer. `serde_json` renders the +/// marker as `null`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Absent; + +impl serde::Serialize for Absent { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_unit_struct(ABSENT_MARKER) + } +} + impl std::fmt::Debug for VirtualClock { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("VirtualClock") diff --git a/rust/src/use_case.rs b/rust/src/use_case.rs index a814e0fa..d301acb7 100644 --- a/rust/src/use_case.rs +++ b/rust/src/use_case.rs @@ -13,8 +13,8 @@ use crate::engine::DialCache; use crate::error::{BoxError, ConfigError, Error}; use crate::identity::{normalize_args, ArgValue, Identity, IntoKeyId}; use crate::operation::{ - downcast_value, erase_load, Comparator, ErasedOperation, Operation, OperationMetadata, Preview, - RecoveryPredicate, SourceBudget, + downcast_value, erase_load, Comparator, ErasedOperation, JsonDefault, MismatchDiffHook, + MismatchEntry, Operation, OperationMetadata, Preview, RecoveryPredicate, SourceBudget, }; use crate::policy::Policy; use crate::scope::Scope; @@ -149,6 +149,7 @@ pub struct UseCaseBuilder { comparator: Option>, should_recover: Option, preview: Option>, + mismatch_diff: Option>, key: Option>, source: Option>, } @@ -179,6 +180,7 @@ where comparator: None, should_recover: None, preview: None, + mismatch_diff: None, key: None, source: None, } @@ -226,9 +228,10 @@ where self } - /// Bounded textual preview of a value for mismatch warnings. - /// Runs through [`Runtime::spawn_blocking`](crate::Runtime::spawn_blocking) - /// after confirmation; callbacks may run on a CPU worker thread. + /// The text mismatch warnings log for each value in place of its JSON; + /// required for value fields under + /// [`register_custom`](Self::register_custom). Difference fields then + /// come only from [`mismatch_diff`](Self::mismatch_diff). See [`Preview`]. pub fn preview( mut self, preview: impl Fn(&T) -> Option + Send + Sync + 'static, @@ -237,6 +240,17 @@ where self } + /// Replace the built-in difference computation of mismatch warnings; + /// required for difference fields under + /// [`register_custom`](Self::register_custom). See [`MismatchDiffHook`]. + pub fn mismatch_diff( + mut self, + hook: impl Fn(&T, &T) -> Vec + Send + Sync + 'static, + ) -> Self { + self.mismatch_diff = Some(Arc::new(hook)); + self + } + /// Select every input dimension that affects the value. Runs only for /// enabled calls. pub fn key(mut self, key: impl Fn(&Args) -> KeySpec + Send + Sync + 'static) -> Self { @@ -260,7 +274,7 @@ where self, codec: Arc>, comparator: Comparator, - preview: Option>, + json: Option>, ) -> Result, ConfigError> { let key = self .key @@ -286,7 +300,12 @@ where codec, comparator, should_recover: self.should_recover, - preview, + // Without a preview of its own, a JSON use case keeps the default. + preview: self + .preview + .or_else(|| json.as_ref().map(JsonDefault::preview)), + mismatch_diff: self.mismatch_diff, + json, } .erase(); Ok(UseCase { @@ -312,8 +331,7 @@ where .comparator .clone() .ok_or_else(|| ConfigError::invalid("DialCache use case needs a comparator"))?; - let preview = self.preview.clone(); - self.finish(codec, comparator, preview) + self.finish(codec, comparator, None) } } @@ -322,19 +340,16 @@ where Args: Clone + Send + Sync + 'static, T: Serialize + DeserializeOwned + PartialEq + Send + Sync + 'static, { - /// Register with the JSON codec, `PartialEq` comparison and JSON previews - /// unless overridden. + /// Register with the JSON codec and `PartialEq` comparison unless + /// overridden. Mismatch warnings use the values' JSON, as for + /// [`Operation::new`]. pub fn register(self) -> Result, ConfigError> { let codec = self.codec.clone().unwrap_or_else(|| Arc::new(JsonCodec)); let comparator = self .comparator .clone() .unwrap_or_else(|| Arc::new(|a: &T, b: &T| a == b)); - let preview = self - .preview - .clone() - .or_else(|| Some(Arc::new(crate::preview::json_preview::))); - self.finish(codec, comparator, preview) + self.finish(codec, comparator, Some(JsonDefault::new())) } } diff --git a/rust/tests/callback_boundaries.rs b/rust/tests/callback_boundaries.rs index 698250e3..3998e8bc 100644 --- a/rust/tests/callback_boundaries.rs +++ b/rust/tests/callback_boundaries.rs @@ -206,6 +206,7 @@ fn synchronous_shadow_source_panic_is_a_source_error_and_releases_capacity() { .policy(Policy::default().remote_ttl_sec(60).shadow(ShadowPolicy { ramp: Some(100.0), log_mismatches: None, + mismatch_logging: None, })); cache .get_or_load( @@ -255,6 +256,7 @@ fn dark_shadow_distinguishes_its_deadline_from_application_timeout_errors() { .shadow(ShadowPolicy { ramp: Some(100.0), log_mismatches: None, + mismatch_logging: None, }), ) .budget(dialcache::SourceBudget::Millis(5)); diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index 39845539..c9f74ef4 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -21,6 +21,8 @@ mod formal; mod frame_vectors; #[path = "formal/key_vectors.rs"] mod key_vectors; +#[path = "formal/mismatch_vectors.rs"] +mod mismatch_vectors; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; @@ -344,6 +346,7 @@ impl Run { "envelopeVectors" => frame_vectors::check_envelope_vector, "compressedDecodeVectors" => frame_vectors::check_compressed_decode_vector, "compressionWriteVectors" => frame_vectors::check_compression_write_vector, + "mismatchPathVectors" => mismatch_vectors::check_mismatch_path_vector, other => return Err(format!("unknown protocol vector group {other}")), }; for vector in vectors { diff --git a/rust/tests/default_logger.rs b/rust/tests/default_logger.rs index 6c7e8a22..cf55231f 100644 --- a/rust/tests/default_logger.rs +++ b/rust/tests/default_logger.rs @@ -1,6 +1,9 @@ -//! Exercise the actual default log facade, including optional value previews. -use dialcache::observe::LogFacadeLogger; -use dialcache::{LogEvent, Logger, ShadowMismatchDetails}; +//! Exercise the actual default log facade, including the optional mismatch fields. +use dialcache::observe::{LogFacadeLogger, ShadowMismatchLoggingOutcome}; +use dialcache::{ + LogEvent, Logger, MismatchDiff, ShadowMismatchDetails, ShadowMismatchDiffDetails, + ShadowMismatchValueDetails, +}; use std::sync::Mutex; struct Capture(Mutex>); @@ -16,27 +19,74 @@ impl log::Log for Capture { static CAPTURE: Capture = Capture(Mutex::new(Vec::new())); #[test] -fn default_warning_includes_only_available_bounded_previews() { +fn default_warning_includes_only_available_bounded_fields() { log::set_logger(&CAPTURE).unwrap(); log::set_max_level(log::LevelFilter::Warn); - for (cached, source) in [ - (Some("\"old\""), Some("\"new\"")), - (None, Some("null")), - (None, None), - ] { - LogFacadeLogger.log(&LogEvent::ShadowMismatch(ShadowMismatchDetails { + let warning = |diff, values, logging_outcome| { + LogEvent::ShadowMismatch(ShadowMismatchDetails { namespace: "app".into(), use_case: "lookup".into(), key_type: "thing".into(), + cached_value_age_seconds: 30.5, cache_key: "one".into(), + diff, + values, + logging_outcome, + }) + }; + let values = |cached: Option<&str>, source: Option<&str>| { + Some(ShadowMismatchValueDetails { cached_value_json: cached.map(str::to_owned), source_value_json: source.map(str::to_owned), - })); + }) + }; + for event in [ + warning(None, values(Some("\"old\""), Some("\"new\"")), None), + warning(None, values(None, Some("null")), None), + warning( + None, + values(None, None), + Some(ShadowMismatchLoggingOutcome::RenderError), + ), + warning( + Some(ShadowMismatchDiffDetails { + level: MismatchDiff::Values, + mismatch_paths: Some(vec!["/a\nb".to_owned(), "/c".to_owned()]), + mismatch_path_count: Some(70), + diff_json: Some(r#"[{"path":"/c","cached":1}]"#.to_owned()), + }), + None, + None, + ), + warning( + Some(ShadowMismatchDiffDetails { + level: MismatchDiff::Paths, + mismatch_paths: None, + mismatch_path_count: None, + diff_json: None, + }), + values(Some("1"), Some("2")), + Some(ShadowMismatchLoggingOutcome::HookError), + ), + ] { + LogFacadeLogger.log(&event); } let messages = CAPTURE.0.lock().unwrap(); - assert!(messages[0].contains("cachedValue=\"old\" sourceValue=\"new\"")); + assert_eq!( + messages[0], + "DialCache shadow validation mismatch: namespace=app useCase=lookup keyType=thing \ + cachedValueAgeSeconds=30.5 cacheKey=one cachedValue=\"old\" sourceValue=\"new\"" + ); assert!(!messages[1].contains("cachedValue=")); assert!(messages[1].contains("sourceValue=null")); + assert!(!messages[1].contains("loggingOutcome")); assert!(!messages[2].contains("cachedValue=")); assert!(!messages[2].contains("sourceValue=")); + assert!(messages[2].ends_with(" loggingOutcome=render_error")); + // Paths are a JSON array, so a control character in a member name stays escaped. + assert!(messages[3].ends_with( + r#" cacheKey=one mismatchPaths=["/a\nb","/c"] mismatchPathCount=70 diffJson=[{"path":"/c","cached":1}]"# + )); + assert!(!messages[4].contains("mismatchPaths=")); + assert!(messages[4].ends_with(" cachedValue=1 sourceValue=2 loggingOutcome=hook_error")); } diff --git a/rust/tests/formal/driver.rs b/rust/tests/formal/driver.rs index 9c3a59e7..574cdbcd 100644 --- a/rust/tests/formal/driver.rs +++ b/rust/tests/formal/driver.rs @@ -7,16 +7,17 @@ use std::collections::HashMap; use std::fmt; use std::sync::Arc; -use dialcache::observe::{Event, LogEvent, Logger, Observer}; +use dialcache::observe::{Event, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails}; use dialcache::protocol::{decode_frame, encode_frame, read_result_from_untrusted_json}; use dialcache::testing::{TestExecutor, VirtualClock}; use dialcache::{ BoxError, Codec, DialCache, Error, FallbackTimeout, Identity, InvalidateRequest, LocalEntry, - LocalRead, LocalStore, LruLocalStore, Operation, Payload, Policy, ReadContext, ReadRequest, - ReadResult, Remote, RuntimePolicy, Scope, SourceBudget, WriteRequest, + LocalRead, LocalStore, LruLocalStore, MismatchDiff, MismatchEntry, Operation, Payload, Policy, + ReadContext, ReadRequest, ReadResult, Remote, RuntimePolicy, Scope, SourceBudget, WriteRequest, }; use futures::future::BoxFuture; use parking_lot::Mutex; +use serde::{Deserialize, Serialize, Serializer}; use serde_json::{json, Map, Value}; use super::causal::{ @@ -59,11 +60,18 @@ pub struct SourceFailure(pub usize); #[error("controlled mutation failure")] pub struct MaintenanceFailure; -/// The cached value domain of the histories: JSON primitives plus absence. +/// The value domain of the histories: JSON primitives, absence, and the +/// source result that has no JSON rendering. #[derive(Debug, Clone)] pub enum Val { + /// The model's absent value, which mismatch warnings treat as an absent + /// side (D13): it serializes as the test-only absent marker. Absent, Json(Value), + /// The `resolve` command's `unrenderable` source result: its + /// serialization fails, so warnings report `render_error` for it, and + /// the driver knows it by its variant, never through JSON. + Unrenderable, } impl Val { @@ -71,32 +79,48 @@ impl Val { Val::Json(value) } - /// The observation encoding: `{absent:true}` for absence, else the primitive. + /// The observation encoding: `{absent:true}` for absence, + /// `{unrenderable:true}` for the unrenderable result, else the primitive. pub fn observed(&self) -> Value { match self { Val::Absent => json!({ "absent": true }), Val::Json(value) => value.clone(), - } - } - - pub fn json_preview(&self) -> Option { - match self { - Val::Absent => None, - Val::Json(value) => serde_json::to_string(value).ok(), + Val::Unrenderable => json!({ "unrenderable": true }), } } } +/// The default comparator: the unrenderable result is unequal to every +/// decoded cached value. impl PartialEq for Val { fn eq(&self, other: &Val) -> bool { match (self, other) { - (Val::Absent, Val::Absent) => true, + (Val::Absent, Val::Absent) | (Val::Unrenderable, Val::Unrenderable) => true, (Val::Json(a), Val::Json(b)) => json_equal(a, b), _ => false, } } } +/// The JSON warnings render: the operation is built with +/// [`Operation::new`], so the built-in walk runs over this rendering. +impl Serialize for Val { + fn serialize(&self, serializer: S) -> Result { + match self { + Val::Absent => dialcache::testing::Absent.serialize(serializer), + Val::Json(value) => value.serialize(serializer), + Val::Unrenderable => Err(serde::ser::Error::custom("controlled unrenderable value")), + } + } +} + +/// Required by [`Operation::new`]; the driver's codec decodes values itself. +impl<'de> Deserialize<'de> for Val { + fn deserialize>(deserializer: D) -> Result { + Value::deserialize(deserializer).map(Val::Json) + } +} + #[derive(Clone)] enum LoaderOutcome { Value(Val), @@ -518,9 +542,13 @@ impl Driver { } "reject" } else { - let value = match input.get("value") { - None => Val::Absent, - Some(value) => Val::from_json(value.clone()), + let value = if input.get("unrenderable") == Some(&Value::Bool(true)) { + Val::Unrenderable + } else { + match input.get("value") { + None => Val::Absent, + Some(value) => Val::from_json(value.clone()), + } }; let gate = s.loaders[index].clone(); gate.settle(LoaderOutcome::Value(value)); @@ -814,13 +842,19 @@ impl Driver { }) } }; - let codec: Arc> = Arc::new(DriverCodec { + // Built with `new`, so warnings render the values' JSON and the + // built-in walk runs; the codec and comparator replace the defaults. + let mut operation = Operation::::new(identity) + .policy(policy) + .budget(self.budget()); + operation.codec = Arc::new(DriverCodec { shared: shared.clone(), }); - let mut operation = Operation::with_codec(identity, codec, comparator) - .policy(policy) - .budget(self.budget()) - .preview(|value: &Val| value.json_preview()); + operation.comparator = Arc::from(comparator); + if self.fixture.get("mismatchDiff") == Some(&Value::from("error")) { + operation = operation + .mismatch_diff(|_: &Val, _: &Val| -> Vec { controlled_panic() }); + } if let Some(mode) = input.get("recovery").and_then(Value::as_str) { operation = operation.should_recover(self.classifier(mode)); } @@ -1205,6 +1239,8 @@ impl Codec for DriverCodec { Ok(match value { Val::Absent => Payload::text("undefined"), Val::Json(json) => Payload::text(serde_json::to_string(&json)?), + // Its dump fails natively; the model never lets it reach a fill. + Val::Unrenderable => return Err("controlled unrenderable value".into()), }) }) } @@ -1478,6 +1514,11 @@ impl Observer for DriverObserver { m.insert("seconds".to_string(), Value::from(*seconds)); ("shadowAge", m) } + Event::ShadowMismatchLogging { labels, outcome } => { + let mut m = outcome_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + ("shadowMismatchLogging", m) + } Event::FutureTimestampOffset { labels, seconds } => { let mut m = labels_map(labels); m.insert("seconds".to_string(), Value::from(*seconds)); @@ -1579,47 +1620,76 @@ struct DriverLogger { shared: Arc>, } +/// The warning's fields as logged, each present exactly when the warning +/// carries it; the `mismatchPaths` list crosses the wire as one JSON string so +/// every observed field stays a primitive (PORTING.md, `mismatchWarning`). +fn warning_observation(details: &ShadowMismatchDetails) -> Map { + let text = |text: &Option| text.clone().map_or(Value::Null, Value::from); + let mut fields = Map::new(); + fields.insert( + "cacheNamespace".to_string(), + Value::from(details.namespace.as_ref()), + ); + fields.insert( + "useCase".to_string(), + Value::from(details.use_case.as_ref()), + ); + fields.insert( + "keyType".to_string(), + Value::from(details.key_type.as_ref()), + ); + fields.insert("outcome".to_string(), Value::from("mismatch")); + fields.insert( + "cachedValueAgeSeconds".to_string(), + Value::from(details.cached_value_age_seconds), + ); + fields.insert( + "cacheKey".to_string(), + Value::from(details.cache_key.as_str()), + ); + if let Some(diff) = &details.diff { + fields.insert( + "mismatchPathsJson".to_string(), + diff.mismatch_paths.as_ref().map_or(Value::Null, |paths| { + Value::from(serde_json::to_string(paths).expect("strings render as JSON")) + }), + ); + fields.insert( + "mismatchPathCount".to_string(), + diff.mismatch_path_count.map_or(Value::Null, Value::from), + ); + // Null at values when the entries are unavailable; at paths only when + // a faulty port carries entries there, which the replay must see. + if diff.level == MismatchDiff::Values || diff.diff_json.is_some() { + fields.insert("diffJson".to_string(), text(&diff.diff_json)); + } + } + if let Some(values) = &details.values { + fields.insert( + "cachedValueJson".to_string(), + text(&values.cached_value_json), + ); + fields.insert( + "sourceValueJson".to_string(), + text(&values.source_value_json), + ); + } + if let Some(outcome) = details.logging_outcome { + fields.insert("loggingOutcome".to_string(), Value::from(outcome.as_str())); + } + fields +} + impl Logger for DriverLogger { + /// Records the warning, then fails as the history's faults say: every + /// call while observers fail, and every warn-level call under + /// `faults.logger`, which leaves the metric observers working. fn log(&self, event: &LogEvent) { let mut s = self.shared.lock(); if let LogEvent::ShadowMismatch(details) = event { - let mut fields = Map::new(); - fields.insert( - "cacheNamespace".to_string(), - Value::from(details.namespace.as_ref()), - ); - fields.insert( - "useCase".to_string(), - Value::from(details.use_case.as_ref()), - ); - fields.insert( - "keyType".to_string(), - Value::from(details.key_type.as_ref()), - ); - fields.insert("outcome".to_string(), Value::from("mismatch")); - fields.insert( - "cacheKey".to_string(), - Value::from(details.cache_key.as_str()), - ); - fields.insert( - "cachedValueJson".to_string(), - details - .cached_value_json - .clone() - .map(Value::from) - .unwrap_or(Value::Null), - ); - fields.insert( - "sourceValueJson".to_string(), - details - .source_value_json - .clone() - .map(Value::from) - .unwrap_or(Value::Null), - ); - s.record("mismatchWarning", fields); + s.record("mismatchWarning", warning_observation(details)); } - let fail = s.observer_fails(); + let fail = s.observer_fails() || (s.fault("logger") && event.level() == LogLevel::Warn); drop(s); if fail { controlled_panic(); diff --git a/rust/tests/formal/inventory.rs b/rust/tests/formal/inventory.rs index bc8847c4..3b39974e 100644 --- a/rust/tests/formal/inventory.rs +++ b/rust/tests/formal/inventory.rs @@ -368,7 +368,7 @@ pub const BEHAVIOR_PROFILE_VERSIONS: [(&str, i64); 16] = [ ("layers", 2), ("recovery", 1), ("independent", 2), - ("shadow", 3), + ("shadow", 4), ("admission", 1), ("dark-layers", 2), ("shadow-read-deadlines", 1), diff --git a/rust/tests/formal/mismatch_vectors.rs b/rust/tests/formal/mismatch_vectors.rs new file mode 100644 index 00000000..02f84a5d --- /dev/null +++ b/rust/tests/formal/mismatch_vectors.rs @@ -0,0 +1,301 @@ +//! Runner for the `mismatchPathVectors` group (W10): the built-in walk over +//! two JSON values, and the difference fields a confirmed-mismatch warning +//! builds from the same pair at each `diff` level. Mirrors the TypeScript +//! `computes mismatch paths:` tests. +//! +//! An omitted `cached` or `source` member is an absent side (D13), which only +//! the test-only [`dialcache::testing::Absent`] marker expresses in Rust; +//! every other value is parsed JSON. Entries compare as parsed JSON, so each +//! side's rendering text stays native. The walk runs through +//! [`diff_shadow_mismatch_json`]. The warning fields come from a real +//! served-hit shadow job whose comparator always reports a mismatch, so every +//! vector reaches a warning that carries exactly its two values. +//! +//! Include from a test binary alongside `fixtures.rs`: +//! `#[path = "formal/fixtures.rs"] mod fixtures;` +//! `#[path = "formal/mismatch_vectors.rs"] mod mismatch_vectors;` + +#![allow(dead_code)] + +use std::future::ready; +use std::sync::Arc; + +use dialcache::observe::{Event, LogEvent, Logger, Observer, ShadowMismatchLoggingOutcome}; +use dialcache::protocol::diff_shadow_mismatch_json; +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, DialCache, Frame, FromSync, Identity, InvalidateRequest, MismatchDiff, + MismatchLogging, Operation, Payload, Policy, ReadContext, ReadRequest, ReadResult, Remote, + ShadowMismatchDetails, ShadowMismatchDiffDetails, ShadowPolicy, SyncCodec, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize, Serializer}; +use serde_json::Value; + +use super::fixtures::assertion_mismatch; + +/// The payload the codec stores for an absent value. +const ABSENT_PAYLOAD: &str = "undefined"; + +/// One compared side: parsed JSON, or absent. +#[derive(Debug, Clone, PartialEq)] +enum Side { + Absent, + Json(Value), +} + +impl Serialize for Side { + fn serialize(&self, serializer: S) -> Result { + match self { + Side::Absent => dialcache::testing::Absent.serialize(serializer), + Side::Json(value) => value.serialize(serializer), + } + } +} + +impl<'de> Deserialize<'de> for Side { + fn deserialize>(deserializer: D) -> Result { + Value::deserialize(deserializer).map(Side::Json) + } +} + +struct SideCodec; + +impl SyncCodec for SideCodec { + fn encode(&self, value: &Side) -> Result { + Ok(match value { + Side::Absent => Payload::text(ABSENT_PAYLOAD), + Side::Json(value) => Payload::text(serde_json::to_string(value)?), + }) + } + + fn decode(&self, payload: Payload) -> Result { + if payload.as_text() == ABSENT_PAYLOAD { + return Ok(Side::Absent); + } + Ok(Side::Json(serde_json::from_slice(&payload.bytes)?)) + } +} + +/// Serves the cached side on every read, the confirmation read included. +struct CachedSide(Payload); + +impl Remote for CachedSide { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(ready(Ok(ReadResult::Hit(Frame { + created_at_ms: WALL_EPOCH_MS as u64, + payload: self.0.clone(), + })))) + } + + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(ready(Ok(()))) + } + + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(ready(Ok(()))) + } +} + +#[derive(Default)] +struct Recorded { + warnings: Mutex>, + outcomes: Mutex>, +} + +struct Recorder(Arc); + +impl Observer for Recorder { + fn observe(&self, event: &Event) { + if let Event::ShadowMismatchLogging { outcome, .. } = event { + self.0.outcomes.lock().push(*outcome); + } + } + + fn observes_shadow_outcomes(&self) -> bool { + true + } +} + +impl Logger for Recorder { + fn log(&self, event: &LogEvent) { + if let LogEvent::ShadowMismatch(details) = event { + self.0.warnings.lock().push(details.clone()); + } + } +} + +/// The side a vector member names: absent when omitted. +fn side(vector: &Value, name: &str) -> Side { + vector.get(name).cloned().map_or(Side::Absent, Side::Json) +} + +/// Parsed-JSON equality: numbers by value, everything else structurally. +fn same_json(a: &Value, b: &Value) -> bool { + match (a, b) { + (Value::Number(x), Value::Number(y)) => x.as_f64() == y.as_f64(), + (Value::Array(x), Value::Array(y)) => { + x.len() == y.len() && x.iter().zip(y).all(|(a, b)| same_json(a, b)) + } + (Value::Object(x), Value::Object(y)) => { + x.len() == y.len() + && x.iter() + .all(|(name, a)| y.get(name).is_some_and(|b| same_json(a, b))) + } + _ => a == b, + } +} + +/// The warnings and logging outcomes of one served-hit call whose shadow job +/// confirms a mismatch between the two sides, logging at `level` with +/// `value` off. +fn warn_once( + cached: &Side, + source: Side, + level: MismatchDiff, +) -> Result< + ( + Vec, + Vec, + ), + String, +> { + let payload = SideCodec + .encode(cached) + .map_err(|error| format!("cached side does not encode: {error}"))?; + let recorded = Arc::new(Recorded::default()); + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote(CachedSide(payload)) + .observer(Recorder(recorded.clone())) + .logger(Recorder(recorded.clone())) + .build() + .map_err(|error| format!("cache does not build: {error}"))?; + let policy = Policy::default().remote_ttl_sec(60).shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: None, + mismatch_logging: Some(MismatchLogging { + value: Some(false), + diff: Some(level), + }), + }); + let mut operation = + Operation::::new(Identity::new("vector", "1", "MismatchPathVector")).policy(policy); + // The JSON rendering stays when the codec and comparator are replaced. + operation.codec = Arc::new(FromSync(SideCodec)); + operation.comparator = Arc::new(|_: &Side, _: &Side| false); + let served = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load(request.scope(), operation, move |_| { + ready(Ok(source.clone())) + }) + .await + }); + executor.drain(); + if let Err(error) = served { + return Err(assertion_mismatch( + "the cached side served", + error.to_string(), + )); + } + let warnings = recorded.warnings.lock().clone(); + let outcomes = recorded.outcomes.lock().clone(); + Ok((warnings, outcomes)) +} + +/// `mismatchPathVectors`: the walk returns `entries` and `count` when both +/// sides are present, and a warning at `diff: "values"` and at +/// `diff: "paths"` carries the entries' paths, the count and, at `values`, +/// the entries, all without a failure. +pub fn check_mismatch_path_vector(vector: &Value) -> Result<(), String> { + let entries = vector + .get("entries") + .and_then(Value::as_array) + .ok_or("entries is not an array")?; + let count = vector + .get("count") + .and_then(Value::as_u64) + .ok_or("count is not an unsigned integer")?; + let paths = entries + .iter() + .map(|entry| { + entry + .get("path") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| format!("entry without a string path: {entry}")) + }) + .collect::, _>>()?; + let (cached, source) = (side(vector, "cached"), side(vector, "source")); + let expected_entries = Value::Array(entries.clone()); + + if let (Side::Json(cached), Side::Json(source)) = (&cached, &source) { + let diff = diff_shadow_mismatch_json(cached, source); + let actual = serde_json::to_value(&diff.entries).map_err(|error| error.to_string())?; + if !same_json(&actual, &expected_entries) || diff.count != count { + return Err(assertion_mismatch( + (&expected_entries, count), + (&actual, diff.count), + )); + } + } + + for level in [MismatchDiff::Values, MismatchDiff::Paths] { + let (warnings, outcomes) = warn_once(&cached, source.clone(), level)?; + if outcomes != [ShadowMismatchLoggingOutcome::Logged] { + return Err(assertion_mismatch( + (level, [ShadowMismatchLoggingOutcome::Logged]), + (level, outcomes), + )); + } + let [warning] = &warnings[..] else { + return Err(assertion_mismatch( + (level, "one warning"), + (level, warnings), + )); + }; + let Some(diff) = &warning.diff else { + return Err(assertion_mismatch( + (level, "difference fields"), + (level, warning), + )); + }; + // At `values` the logged entries must parse to the vector's entries; + // at `paths` no entries are logged. + let entries_match = match (&diff.diff_json, level) { + (Some(text), MismatchDiff::Values) => serde_json::from_str::(text) + .is_ok_and(|logged| same_json(&logged, &expected_entries)), + (None, MismatchDiff::Paths) => true, + _ => false, + }; + let expected = ShadowMismatchDiffDetails { + level, + mismatch_paths: Some(paths.clone()), + mismatch_path_count: Some(count), + diff_json: diff.diff_json.clone(), + }; + if !entries_match + || diff != &expected + || warning.values.is_some() + || warning.logging_outcome.is_some() + { + let expected_entries = (level == MismatchDiff::Values).then_some(&expected_entries); + return Err(assertion_mismatch( + (level, &paths, count, expected_entries), + ( + level, + &diff.mismatch_paths, + diff.mismatch_path_count, + &diff.diff_json, + &warning.values, + warning.logging_outcome, + ), + )); + } + } + Ok(()) +} diff --git a/rust/tests/metrics_exporters.rs b/rust/tests/metrics_exporters.rs index 92be3d1d..3149ad76 100644 --- a/rust/tests/metrics_exporters.rs +++ b/rust/tests/metrics_exporters.rs @@ -7,7 +7,8 @@ use std::sync::Arc; use dialcache::datadog::{metric_suffix, DatadogError}; use dialcache::observe::{ CoalescingScope, CompressionOperation, CompressionOutcome, DisabledReason, ErrorKind, Labels, - Layer, MissReason, OutcomeLabels, RecoveryOutcome, SerializationOperation, ShadowOutcome, + Layer, MissReason, OutcomeLabels, RecoveryOutcome, SerializationOperation, + ShadowMismatchLoggingOutcome, ShadowOutcome, }; use dialcache::{ DatadogObserver, DatadogOptions, DogStatsdClient, Event, MetricKind, ObservationMetricType, @@ -68,6 +69,10 @@ fn metric_test_event(kind: MetricKind) -> Event { outcome: ShadowOutcome::Mismatch, seconds: 0.25, }, + MetricKind::ShadowMismatchLogging => Event::ShadowMismatchLogging { + labels: outcome(), + outcome: ShadowMismatchLoggingOutcome::Logged, + }, MetricKind::FutureTimestampOffset => Event::FutureTimestampOffset { labels: base(), seconds: 0.25, @@ -179,7 +184,7 @@ fn datadog_metric_names_units_and_labels() { assert!(observer.observes_shadow_outcomes()); // (kind, suffix, method, value), copied from go/metrics_test.go. - let cases: [(MetricKind, &str, &str, f64); 19] = [ + let cases: [(MetricKind, &str, &str, f64); 20] = [ (MetricKind::Request, "request.count", "increment", 1.0), (MetricKind::Miss, "miss.count", "increment", 1.0), (MetricKind::Disabled, "disabled.count", "increment", 1.0), @@ -203,6 +208,12 @@ fn datadog_metric_names_units_and_labels() { "distribution", 0.25, ), + ( + MetricKind::ShadowMismatchLogging, + "shadow_mismatch_logging.count", + "increment", + 1.0, + ), ( MetricKind::FutureTimestampOffset, "future_timestamp_offset", @@ -349,17 +360,25 @@ fn datadog_rejects_invalid_namespaces_and_long_names() { Some(DatadogError::InvalidNamespace(String::new())), "accepted explicitly empty namespace" ); - // The longest suffix, "stale_recovery.value_age", is 24 characters; with - // the dot a 175-character namespace reaches exactly 200 and 176 exceeds it. - let longest = "stale_recovery.value_age"; - assert_eq!(longest.len(), 24); - let fits = "a".repeat(175); + // The longest suffix, "shadow_mismatch_logging.count", is 29 characters; + // with the dot a 170-character namespace reaches exactly 200 and 171 + // exceeds it. + let longest = "shadow_mismatch_logging.count"; + assert_eq!(longest.len(), 29); + assert_eq!( + MetricKind::ALL + .iter() + .map(|kind| metric_suffix(*kind).len()) + .max(), + Some(longest.len()) + ); + let fits = "a".repeat(170); DatadogObserver::new( RecordingClient::default(), DatadogOptions::new(ObservationMetricType::Distribution).namespace(fits.clone()), ) .expect("200-character name fits"); - let overflow = "a".repeat(176); + let overflow = "a".repeat(171); let error = DatadogObserver::new( RecordingClient::default(), DatadogOptions::new(ObservationMetricType::Distribution).namespace(overflow.clone()), @@ -404,7 +423,7 @@ mod prometheus_exporter { const OUTCOME: &[&str] = &["cache_namespace", "use_case", "key_type", "outcome"]; /// Copied from go/metrics_prometheus.go PrometheusCollectorSchemas, in order. - const EXPECTED: [Expected; 19] = [ + const EXPECTED: [Expected; 20] = [ Expected { kind: "disabled", name: "dialcache_disabled_counter", @@ -468,6 +487,13 @@ mod prometheus_exporter { labels: OUTCOME, buckets: AGE, }, + Expected { + kind: "shadowMismatchLogging", + name: "dialcache_shadow_mismatch_logging_counter", + help: "DialCache confirmed shadow-mismatch warning outcomes.", + labels: OUTCOME, + buckets: &[], + }, Expected { kind: "futureTimestampOffset", name: "dialcache_future_timestamp_offset_histogram", @@ -556,7 +582,7 @@ mod prometheus_exporter { fn wire_schema_matches_the_reference_table() { for prefix in ["", "svc_"] { let actual: Vec = schemas(prefix); - assert_eq!(actual.len(), 19); + assert_eq!(actual.len(), 20); for (schema, expected) in actual.iter().zip(EXPECTED.iter()) { assert_eq!(schema.kind.as_str(), expected.kind); assert_eq!(schema.name, format!("{prefix}{}", expected.name)); @@ -591,7 +617,7 @@ mod prometheus_exporter { observer.observe(&metric_test_event(kind)); } let families = registry.gather(); - assert_eq!(families.len(), 19); + assert_eq!(families.len(), 20); for expected in EXPECTED.iter() { let name = format!("scrape_{}", expected.name); let family = families diff --git a/rust/tests/mismatch_walk_cost.rs b/rust/tests/mismatch_walk_cost.rs new file mode 100644 index 00000000..2c50bcab --- /dev/null +++ b/rust/tests/mismatch_walk_cost.rs @@ -0,0 +1,95 @@ +//! Pins the built-in mismatch walk's pointer cost: it builds pointer text +//! only for the entries it retains, not for every difference it counts, and +//! only as far as the pointer cap reads, so a deep chain of long member names +//! over a wide array costs no copying per visited member. +//! +//! A counting allocator measures the bytes the walk allocates on the test's +//! own thread, so the bound holds however fast the machine is. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; + +use dialcache::preview::{SHADOW_MISMATCH_MAX_ENTRIES, SHADOW_MISMATCH_PATH_MAX_BYTES}; +use dialcache::protocol::diff_shadow_mismatch_json; +use serde_json::{json, Value}; + +struct CountingAllocator; + +thread_local! { + static ALLOCATED: Cell = const { Cell::new(0) }; +} + +fn count(bytes: usize) { + let _ = ALLOCATED.try_with(|allocated| allocated.set(allocated.get() + bytes)); +} + +// SAFETY: every call forwards to the system allocator unchanged. +unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + count(layout.size()); + System.alloc(layout) + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + count(layout.size()); + System.alloc_zeroed(layout) + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + count(new_size); + System.realloc(ptr, layout, new_size) + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + System.dealloc(ptr, layout) + } +} + +#[global_allocator] +static ALLOCATOR: CountingAllocator = CountingAllocator; + +fn allocated_by(work: impl FnOnce() -> R) -> (R, usize) { + let before = ALLOCATED.with(Cell::get); + let result = work(); + (result, ALLOCATED.with(Cell::get) - before) +} + +/// 31 levels of one 1 KiB member name over a 10,000-element array, so every +/// array element sits under a 31 KiB pointer prefix. +fn long_name_chain(items: Vec) -> Value { + let name = "k".repeat(1024); + let mut value = Value::Array(items); + for _ in 0..31 { + value = json!({ name.clone(): value }); + } + value +} + +#[test] +fn pointer_text_is_built_only_for_retained_entries_and_only_up_to_the_cap() { + const ITEMS: usize = 10_000; + // Far more differences than the walk retains, so text built for each + // counted difference costs well over the bound below. + const DIFFERING: usize = 5_000; + let cached = long_name_chain((0..ITEMS).map(|index| json!(index)).collect()); + let source = long_name_chain( + (0..ITEMS) + .map(|index| json!(if index < ITEMS - DIFFERING { index } else { 0 })) + .collect(), + ); + + let (diff, bytes) = allocated_by(|| diff_shadow_mismatch_json(&cached, &source)); + + assert_eq!(diff.count, DIFFERING as u64); + assert_eq!(diff.entries.len(), SHADOW_MISMATCH_MAX_ENTRIES); + assert!(diff + .entries + .iter() + .all(|entry| entry.path.len() <= SHADOW_MISMATCH_PATH_MAX_BYTES + && entry.path.starts_with("/kkk"))); + // Copying the prefix for every visited member allocates over 300 MB here, + // building a capped pointer for every counted difference about 10 MB, and + // building each retained pointer whole about 4 MB; the walk itself needs + // well under 1 MiB. + assert!(bytes < 1 << 20, "the walk allocated {bytes} bytes"); +} diff --git a/rust/tests/protocol_frames.rs b/rust/tests/protocol_frames.rs index 81165114..d31373f9 100644 --- a/rust/tests/protocol_frames.rs +++ b/rust/tests/protocol_frames.rs @@ -1,6 +1,6 @@ -//! Frame, decode, timestamp, duration and envelope conformance against the -//! portable protocol corpus (`formal/catalogs/protocol-vectors.json` plus the -//! Quint-generated frame and envelope artifacts). +//! Frame, decode, timestamp, duration, envelope and mismatch-path conformance +//! against the portable protocol corpus (`formal/catalogs/protocol-vectors.json` +//! plus the Quint-generated frame and envelope artifacts). #[path = "formal/digest.rs"] mod digest; @@ -10,6 +10,8 @@ mod digest; mod fixtures; #[path = "formal/frame_vectors.rs"] mod frame_vectors; +#[path = "formal/mismatch_vectors.rs"] +mod mismatch_vectors; use serde_json::{json, Value}; @@ -29,8 +31,8 @@ fn frame_assertions_distinguish_library_rejections_from_bad_fixtures() { assert!(!error.contains("PROTOCOL_ASSERTION_FAILURE"), "{error}"); } -/// The eight groups this test owns, with their fixed-corpus sizes. -const GROUPS: [(&str, usize, Check); 8] = [ +/// The nine groups this test owns, with their fixed-corpus sizes. +const GROUPS: [(&str, usize, Check); 9] = [ ("frameVectors", 8, frame_vectors::check_frame_vector), ( "trackedDecodeVectors", @@ -59,6 +61,11 @@ const GROUPS: [(&str, usize, Check); 8] = [ 6, frame_vectors::check_compression_write_vector, ), + ( + "mismatchPathVectors", + 57, + mismatch_vectors::check_mismatch_path_vector, + ), ]; #[test] diff --git a/rust/tests/shadow_mismatch_logging.rs b/rust/tests/shadow_mismatch_logging.rs new file mode 100644 index 00000000..67e06856 --- /dev/null +++ b/rust/tests/shadow_mismatch_logging.rs @@ -0,0 +1,1098 @@ +//! Confirmed-mismatch warnings end to end: the fields each mismatch-logging +//! leaf adds, the hook, the `ShadowMismatchLogging` outcome and its +//! precedence, logger failures, absent values and runtime leaf resolution. + +use std::future::{ready, Future}; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use dialcache::observe::{ErrorKind, Layer, ShadowMismatchLoggingOutcome, ShadowOutcome}; +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, DialCache, Event, Frame, FromSync, Identity, InvalidateRequest, JsonCodec, KeySpec, + LogEvent, Logger, MismatchDiff, MismatchEntry, MismatchLogging, Observer, Operation, Payload, + Policy, ReadContext, ReadRequest, ReadResult, Remote, RuntimePolicy, ShadowMismatchDetails, + ShadowMismatchDiffDetails, ShadowMismatchValueDetails, ShadowPolicy, SyncCodec, WriteRequest, +}; +use futures::channel::oneshot; +use futures::future::BoxFuture; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize, Serializer}; +use serde_json::{json, Value}; + +const AGE_MS: u64 = 30_500; +const CACHED_JSON: &str = r#"{"id":"123","version":1}"#; +const SOURCE_JSON: &str = r#"{"id":"123","version":2}"#; +const DIFF_JSON: &str = r#"[{"path":"/version","cached":1,"source":2}]"#; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +struct Record { + id: String, + version: u64, +} + +fn record(version: u64) -> Record { + Record { + id: "123".into(), + version, + } +} + +/// Every read returns the same frame, so the confirmation read finds the +/// payload unchanged. +struct SnapshotRemote(String); + +impl Remote for SnapshotRemote { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(ready(Ok(ReadResult::Hit(Frame { + created_at_ms: WALL_EPOCH_MS as u64 - AGE_MS, + payload: Payload::text(self.0.clone()), + })))) + } + + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(ready(Ok(()))) + } + + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(ready(Ok(()))) + } +} + +/// One step of the shared diagnostic sequence. +#[derive(Debug, Clone, PartialEq)] +enum Step { + Event(Event), + Warning(Box), +} + +#[derive(Default)] +struct Recorder { + steps: Mutex>, + logger_panics: bool, +} + +impl Recorder { + fn events(&self) -> Vec { + self.steps + .lock() + .iter() + .filter_map(|step| match step { + Step::Event(event) => Some(event.clone()), + Step::Warning(_) => None, + }) + .collect() + } + + fn warnings(&self) -> Vec { + self.steps + .lock() + .iter() + .filter_map(|step| match step { + Step::Warning(details) => Some(*details.clone()), + Step::Event(_) => None, + }) + .collect() + } + + fn verdicts(&self) -> Vec { + self.events() + .into_iter() + .filter_map(|event| match event { + Event::ShadowValidation { outcome, .. } => Some(outcome), + _ => None, + }) + .collect() + } + + fn logging_outcomes(&self) -> Vec { + self.events() + .into_iter() + .filter_map(|event| match event { + Event::ShadowMismatchLogging { outcome, .. } => Some(outcome), + _ => None, + }) + .collect() + } + + fn config_resolutions(&self) -> usize { + self.events() + .iter() + .filter(|event| { + matches!( + event, + Event::Error { + error: ErrorKind::ConfigResolution, + labels, + in_fallback: false, + } if labels.layer == Layer::Remote + ) + }) + .count() + } + + fn age_seconds(&self) -> Vec { + self.events() + .into_iter() + .filter_map(|event| match event { + Event::ShadowValueAge { seconds, .. } => Some(seconds), + _ => None, + }) + .collect() + } +} + +struct RecordingObserver(Arc); + +impl Observer for RecordingObserver { + fn observe(&self, event: &Event) { + self.0.steps.lock().push(Step::Event(event.clone())); + } + + fn observes_shadow_outcomes(&self) -> bool { + true + } +} + +struct RecordingLogger(Arc); + +impl Logger for RecordingLogger { + fn log(&self, event: &LogEvent) { + if let LogEvent::ShadowMismatch(details) = event { + self.0 + .steps + .lock() + .push(Step::Warning(Box::new(details.clone()))); + if self.0.logger_panics { + panic!("logger unavailable"); + } + } + } +} + +type Overlay = Option; + +fn cache(executor: &TestExecutor, recorder: &Arc, overlay: Overlay) -> DialCache { + let mut builder = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote(SnapshotRemote(CACHED_JSON.to_string())) + .observer(RecordingObserver(recorder.clone())) + .logger(RecordingLogger(recorder.clone())); + if let Some(overlay) = overlay { + builder = builder.policy_provider(move |_| { + let overlay = overlay.clone(); + async move { Ok(Some(RuntimePolicy::from_json(overlay))) } + }); + } + builder.build().expect("valid configuration") +} + +fn shadow(log_mismatches: Option, logging: Option) -> Policy { + Policy::default().remote_ttl_sec(60).shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches, + mismatch_logging: logging, + }) +} + +fn leaves(value: Option, diff: Option) -> Option { + Some(MismatchLogging { value, diff }) +} + +fn operation(use_case: &str, policy: Policy) -> Operation { + Operation::new(Identity::new("user_id", "123", use_case)).policy(policy) +} + +/// One served-hit call whose shadow job confirms a mismatch, run to quiescence. +fn run( + recorder: &Arc, + overlay: Overlay, + operation: Operation, + source: T, +) -> Arc { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = cache(&executor, recorder, overlay); + let result = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load(request.scope(), operation, move |_| { + let source = source.clone(); + async move { Ok(source) } + }) + .await + }); + executor.drain(); + result.expect("the caller is served") +} + +fn confirmed(use_case: &str, policy: Policy, overlay: Overlay) -> Arc { + let recorder = Arc::new(Recorder::default()); + let served = run(&recorder, overlay, operation(use_case, policy), record(2)); + assert_eq!(*served, record(1)); + assert_eq!(recorder.verdicts(), vec![ShadowOutcome::Mismatch]); + recorder +} + +fn paths(level: MismatchDiff, diff_json: Option<&str>) -> Option { + Some(ShadowMismatchDiffDetails { + level, + mismatch_paths: Some(vec!["/version".to_string()]), + mismatch_path_count: Some(1), + diff_json: diff_json.map(str::to_owned), + }) +} + +fn failed_diff(level: MismatchDiff) -> Option { + Some(ShadowMismatchDiffDetails { + level, + mismatch_paths: None, + mismatch_path_count: None, + diff_json: None, + }) +} + +fn values(cached: Option<&str>, source: Option<&str>) -> Option { + Some(ShadowMismatchValueDetails { + cached_value_json: cached.map(str::to_owned), + source_value_json: source.map(str::to_owned), + }) +} + +/// The one warning, checked against the expected content fields. +fn assert_warning( + recorder: &Recorder, + use_case: &str, + diff: Option, + values: Option, + logging_outcome: Option, +) { + assert_eq!( + recorder.warnings(), + vec![ShadowMismatchDetails { + namespace: "urn".into(), + use_case: use_case.into(), + key_type: "user_id".into(), + cached_value_age_seconds: AGE_MS as f64 / 1000.0, + cache_key: format!("urn:user_id:123#{use_case}"), + diff, + values, + logging_outcome, + }] + ); +} + +#[test] +fn no_warning_or_logging_event_while_logging_is_omitted() { + let recorder = confirmed("LoggingOff", shadow(None, None), None); + assert!(recorder.warnings().is_empty()); + assert!(recorder.logging_outcomes().is_empty()); +} + +#[test] +fn diff_paths_logs_only_the_paths_and_their_count() { + let recorder = confirmed( + "PathsOnly", + shadow(None, leaves(None, Some(MismatchDiff::Paths))), + None, + ); + assert_warning( + &recorder, + "PathsOnly", + paths(MismatchDiff::Paths, None), + None, + None, + ); + assert_eq!( + recorder.logging_outcomes(), + vec![ShadowMismatchLoggingOutcome::Logged] + ); + // The warning carries the age the verdict's metric observed. + assert_eq!(recorder.age_seconds(), vec![30.5]); +} + +#[test] +fn diff_values_without_value_logs_the_entries_without_whole_values() { + let recorder = confirmed( + "ValuesDiffOnly", + shadow(None, leaves(Some(false), Some(MismatchDiff::Values))), + None, + ); + assert_warning( + &recorder, + "ValuesDiffOnly", + paths(MismatchDiff::Values, Some(DIFF_JSON)), + None, + None, + ); +} + +#[test] +fn value_alone_logs_the_whole_values_and_never_runs_the_hook() { + let recorder = Arc::new(Recorder::default()); + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + let operation = operation("ValueOnly", shadow(None, leaves(Some(true), None))).mismatch_diff( + move |_: &Record, _: &Record| { + seen.fetch_add(1, Ordering::SeqCst); + Vec::new() + }, + ); + run(&recorder, None, operation, record(2)); + assert_warning( + &recorder, + "ValueOnly", + None, + values(Some(CACHED_JSON), Some(SOURCE_JSON)), + None, + ); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn the_shorthand_logs_every_field_in_the_reference_order() { + let recorder = confirmed("Shorthand", shadow(Some(true), None), None); + assert_warning( + &recorder, + "Shorthand", + paths(MismatchDiff::Values, Some(DIFF_JSON)), + values(Some(CACHED_JSON), Some(SOURCE_JSON)), + None, + ); + let warning = LogEvent::ShadowMismatch(recorder.warnings().remove(0)); + assert_eq!( + warning.to_string(), + format!( + "DialCache shadow validation mismatch: namespace=urn useCase=Shorthand keyType=user_id \ + cachedValueAgeSeconds=30.5 cacheKey=urn:user_id:123#Shorthand \ + mismatchPaths=[\"/version\"] mismatchPathCount=1 diffJson={DIFF_JSON} \ + cachedValue={CACHED_JSON} sourceValue={SOURCE_JSON}" + ) + ); +} + +#[test] +fn the_logging_event_follows_the_verdict_and_the_warning_and_a_match_emits_none() { + let recorder = Arc::new(Recorder::default()); + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = cache(&executor, &recorder, None); + for version in [1, 2, 3] { + let cache = cache.clone(); + executor.block_on(async move { + let request = cache.enable_guard(); + let operation = operation( + "LoggingOrder", + shadow(None, leaves(None, Some(MismatchDiff::Paths))), + ); + cache + .get_or_load(request.scope(), operation, move |_| { + ready(Ok(record(version))) + }) + .await + .unwrap(); + }); + executor.drain(); + } + assert_eq!( + recorder.verdicts(), + vec![ + ShadowOutcome::Match, + ShadowOutcome::Mismatch, + ShadowOutcome::Mismatch + ] + ); + let sequence: Vec<&'static str> = recorder + .steps + .lock() + .iter() + .filter_map(|step| match step { + Step::Event(Event::ShadowValidation { .. }) => Some("verdict"), + Step::Event(Event::ShadowValueAge { .. }) => Some("age"), + Step::Warning(_) => Some("warning"), + Step::Event(Event::ShadowMismatchLogging { .. }) => Some("logging"), + _ => None, + }) + .collect(); + assert_eq!( + sequence, + [ + "verdict", "age", "verdict", "age", "warning", "logging", "verdict", "age", "warning", + "logging" + ] + ); +} + +#[test] +fn the_hook_replaces_the_built_in_walk_for_inline_and_registered_use_cases() { + let hook = |cached: &Record, source: &Record| { + assert_eq!((cached, source), (&record(1), &record(2))); + vec![ + MismatchEntry { + path: "/custom".into(), + cached: Some(json!("c")), + source: Some(json!("s")), + }, + MismatchEntry { + path: "/only-path".into(), + cached: None, + source: None, + }, + MismatchEntry { + path: "/one-side".into(), + cached: None, + source: Some(json!({"nested": true})), + }, + ] + }; + let expected = Some(ShadowMismatchDiffDetails { + level: MismatchDiff::Values, + mismatch_paths: Some(vec![ + "/custom".to_string(), + "/only-path".to_string(), + "/one-side".to_string(), + ]), + mismatch_path_count: Some(3), + diff_json: Some( + r#"[{"path":"/custom","cached":"c","source":"s"},{"path":"/only-path"},{"path":"/one-side","source":{"nested":true}}]"# + .to_string(), + ), + }); + + let recorder = Arc::new(Recorder::default()); + let inline = operation("HookInline", shadow(Some(true), None)).mismatch_diff(hook); + run(&recorder, None, inline, record(2)); + assert_warning( + &recorder, + "HookInline", + expected.clone(), + values(Some(CACHED_JSON), Some(SOURCE_JSON)), + None, + ); + + let recorder = Arc::new(Recorder::default()); + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = cache(&executor, &recorder, None); + let use_case = cache + .use_case::<(), Record>("user_id", "HookRegistered") + .policy(shadow(Some(true), None)) + .key(|_| KeySpec::new("123")) + .source(|_, ()| ready(Ok(record(2)))) + .mismatch_diff(hook) + .register() + .unwrap(); + executor.block_on(async move { + let request = cache.enable_guard(); + use_case.get(request.scope(), ()).await.unwrap(); + }); + executor.drain(); + assert_warning( + &recorder, + "HookRegistered", + expected, + values(Some(CACHED_JSON), Some(SOURCE_JSON)), + None, + ); + assert_eq!( + recorder.logging_outcomes(), + vec![ShadowMismatchLoggingOutcome::Logged] + ); +} + +#[test] +fn a_panicking_hook_logs_hook_error_without_touching_the_verdict_or_the_caller() { + let recorder = Arc::new(Recorder::default()); + let operation = operation("HookPanics", shadow(Some(true), None)) + .mismatch_diff(|_: &Record, _: &Record| panic!("diff hook unavailable")); + assert_eq!(*run(&recorder, None, operation, record(2)), record(1)); + assert_eq!(recorder.verdicts(), vec![ShadowOutcome::Mismatch]); + assert_warning( + &recorder, + "HookPanics", + failed_diff(MismatchDiff::Values), + values(Some(CACHED_JSON), Some(SOURCE_JSON)), + Some(ShadowMismatchLoggingOutcome::HookError), + ); + assert_eq!( + recorder.logging_outcomes(), + vec![ShadowMismatchLoggingOutcome::HookError] + ); +} + +#[test] +fn a_panicking_logger_records_logger_error_and_leaves_the_caller_unaffected() { + let recorder = Arc::new(Recorder { + logger_panics: true, + ..Recorder::default() + }); + let operation = operation("LoggerPanics", shadow(Some(true), None)); + assert_eq!(*run(&recorder, None, operation, record(2)), record(1)); + assert_eq!(recorder.verdicts(), vec![ShadowOutcome::Mismatch]); + assert_warning( + &recorder, + "LoggerPanics", + paths(MismatchDiff::Values, Some(DIFF_JSON)), + values(Some(CACHED_JSON), Some(SOURCE_JSON)), + None, + ); + assert_eq!( + recorder.logging_outcomes(), + vec![ShadowMismatchLoggingOutcome::LoggerError] + ); +} + +/// A source value whose serialization fails, as the replay's unrenderable +/// source input does. +#[derive(Debug, Clone, PartialEq)] +enum Unrenderable { + Record(Record), + Fails, +} + +impl Serialize for Unrenderable { + fn serialize(&self, serializer: S) -> Result { + match self { + Unrenderable::Record(record) => record.serialize(serializer), + Unrenderable::Fails => Err(serde::ser::Error::custom("not serializable")), + } + } +} + +impl<'de> Deserialize<'de> for Unrenderable { + fn deserialize>(deserializer: D) -> Result { + Record::deserialize(deserializer).map(Unrenderable::Record) + } +} + +#[test] +fn hook_error_and_render_error_rank_above_a_failing_logger() { + for (name, hook_panics, outcome) in [ + ( + "HookAndLoggerFail", + true, + ShadowMismatchLoggingOutcome::HookError, + ), + ( + "RenderAndLoggerFail", + false, + ShadowMismatchLoggingOutcome::RenderError, + ), + ] { + let recorder = Arc::new(Recorder { + logger_panics: true, + ..Recorder::default() + }); + let mut operation = Operation::::new(Identity::new("user_id", "123", name)) + .policy(shadow(Some(true), None)); + if hook_panics { + operation = operation.mismatch_diff(|_, _| panic!("hook failed")); + } + let served = run(&recorder, None, operation, Unrenderable::Fails); + assert_eq!(*served, Unrenderable::Record(record(1))); + assert_warning( + &recorder, + name, + failed_diff(MismatchDiff::Values), + values(Some(CACHED_JSON), None), + Some(outcome), + ); + assert_eq!(recorder.logging_outcomes(), vec![outcome]); + } +} + +#[test] +fn an_unrenderable_source_value_logs_render_error_with_its_fields_null() { + let recorder = Arc::new(Recorder::default()); + let operation = Operation::::new(Identity::new("user_id", "123", "Unrenderable")) + .policy(shadow(None, leaves(Some(true), Some(MismatchDiff::Paths)))); + run(&recorder, None, operation, Unrenderable::Fails); + assert_warning( + &recorder, + "Unrenderable", + failed_diff(MismatchDiff::Paths), + values(Some(CACHED_JSON), None), + Some(ShadowMismatchLoggingOutcome::RenderError), + ); + assert_eq!( + recorder.logging_outcomes(), + vec![ShadowMismatchLoggingOutcome::RenderError] + ); +} + +/// A value the default codec cannot hold: the test-only absent marker stands +/// for TypeScript's top-level `undefined`. +#[derive(Debug, Clone, PartialEq)] +enum Maybe { + Absent, + Present(Record), +} + +impl Serialize for Maybe { + fn serialize(&self, serializer: S) -> Result { + match self { + Maybe::Absent => dialcache::testing::Absent.serialize(serializer), + Maybe::Present(record) => record.serialize(serializer), + } + } +} + +impl<'de> Deserialize<'de> for Maybe { + fn deserialize>(deserializer: D) -> Result { + Record::deserialize(deserializer).map(Maybe::Present) + } +} + +struct MaybeCodec; + +impl SyncCodec for MaybeCodec { + fn encode(&self, value: &Maybe) -> Result { + match value { + Maybe::Absent => Ok(Payload::text("__absent__")), + Maybe::Present(record) => JsonCodec::encode_value(record), + } + } + + fn decode(&self, payload: Payload) -> Result { + if payload.as_text() == "__absent__" { + return Ok(Maybe::Absent); + } + JsonCodec::decode_value(&payload).map(Maybe::Present) + } +} + +#[test] +fn an_absent_cached_value_is_an_absent_side_not_a_render_failure() { + let recorder = Arc::new(Recorder::default()); + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote(SnapshotRemote("__absent__".to_string())) + .observer(RecordingObserver(recorder.clone())) + .logger(RecordingLogger(recorder.clone())) + .build() + .unwrap(); + let mut operation = Operation::::new(Identity::new("user_id", "123", "AbsentCached")) + .policy(shadow(Some(true), None)); + // The JSON rendering stays when the codec is replaced. + operation.codec = Arc::new(FromSync(MaybeCodec)); + let served = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load(request.scope(), operation, |_| { + ready(Ok(Maybe::Present(record(2)))) + }) + .await + }); + executor.drain(); + assert_eq!(*served.unwrap(), Maybe::Absent); + assert_eq!(recorder.verdicts(), vec![ShadowOutcome::Mismatch]); + assert_warning( + &recorder, + "AbsentCached", + Some(ShadowMismatchDiffDetails { + level: MismatchDiff::Values, + mismatch_paths: Some(vec![String::new()]), + mismatch_path_count: Some(1), + diff_json: Some(format!(r#"[{{"path":"","source":{SOURCE_JSON}}}]"#)), + }), + values(None, Some(SOURCE_JSON)), + None, + ); + assert_eq!( + recorder.logging_outcomes(), + vec![ShadowMismatchLoggingOutcome::Logged] + ); +} + +struct TextCodec; + +impl SyncCodec for TextCodec { + fn encode(&self, text: &String) -> Result { + Ok(Payload::text(text.clone())) + } + + fn decode(&self, payload: Payload) -> Result { + Ok(payload.as_text().into_owned()) + } +} + +#[test] +fn a_custom_codec_operation_logs_diff_fields_only_through_its_hook() { + let custom = |name: &str| { + Operation::with_codec( + Identity::new("user_id", "123", name), + Arc::new(FromSync(TextCodec)), + |a: &String, b: &String| a == b, + ) + .policy(shadow(Some(true), None)) + }; + let recorder = Arc::new(Recorder::default()); + run(&recorder, None, custom("CustomCodec"), "source".to_string()); + assert_warning( + &recorder, + "CustomCodec", + failed_diff(MismatchDiff::Values), + values(None, None), + Some(ShadowMismatchLoggingOutcome::RenderError), + ); + + let recorder = Arc::new(Recorder::default()); + let operation = custom("CustomCodecHook") + .preview(|text: &String| Some(format!("<{text}>"))) + .mismatch_diff(|_: &String, _: &String| { + vec![MismatchEntry { + path: String::new(), + cached: None, + source: None, + }] + }); + run(&recorder, None, operation, "source".to_string()); + assert_warning( + &recorder, + "CustomCodecHook", + Some(ShadowMismatchDiffDetails { + level: MismatchDiff::Values, + mismatch_paths: Some(vec![String::new()]), + mismatch_path_count: Some(1), + diff_json: Some(r#"[{"path":""}]"#.to_string()), + }), + values(Some(&format!("<{CACHED_JSON}>")), Some("")), + None, + ); +} + +#[test] +fn a_preview_replaces_the_json_rendering_so_the_walk_never_logs_what_it_redacts() { + let redact = |record: &Record| Some(format!(r#"{{"id":"{}"}}"#, record.id)); + let redacted = values(Some(r#"{"id":"123"}"#), Some(r#"{"id":"123"}"#)); + let recorder = Arc::new(Recorder::default()); + let inline = operation("RedactedInline", shadow(Some(true), None)).preview(redact); + run(&recorder, None, inline, record(2)); + assert_warning( + &recorder, + "RedactedInline", + failed_diff(MismatchDiff::Values), + redacted.clone(), + Some(ShadowMismatchLoggingOutcome::RenderError), + ); + assert_eq!( + recorder.logging_outcomes(), + vec![ShadowMismatchLoggingOutcome::RenderError] + ); + + // A hook supplies the difference fields, here for a registered use case. + let recorder = Arc::new(Recorder::default()); + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = cache(&executor, &recorder, None); + let use_case = cache + .use_case::<(), Record>("user_id", "RedactedRegistered") + .policy(shadow(Some(true), None)) + .key(|_| KeySpec::new("123")) + .source(|_, ()| ready(Ok(record(2)))) + .preview(redact) + .mismatch_diff(|_: &Record, _: &Record| { + vec![MismatchEntry { + path: "/version".into(), + cached: None, + source: None, + }] + }) + .register() + .unwrap(); + executor.block_on(async move { + let request = cache.enable_guard(); + use_case.get(request.scope(), ()).await.unwrap(); + }); + executor.drain(); + assert_warning( + &recorder, + "RedactedRegistered", + paths(MismatchDiff::Values, Some(r#"[{"path":"/version"}]"#)), + redacted, + None, + ); +} + +#[test] +fn an_explicit_none_preview_logs_no_values_and_no_built_in_differences() { + // As on `main`, `preview = None` logs no values. It also removes the JSON + // rendering, so the operation behaves like one built with `with_codec`. + for (name, logging, diff) in [ + ( + "NoPreviewShorthand", + shadow(Some(true), None), + failed_diff(MismatchDiff::Values), + ), + ( + "NoPreviewValue", + shadow(None, leaves(Some(true), None)), + None, + ), + ( + "NoPreviewPaths", + shadow(None, leaves(Some(true), Some(MismatchDiff::Paths))), + failed_diff(MismatchDiff::Paths), + ), + ] { + let recorder = Arc::new(Recorder::default()); + let mut without_preview = operation(name, logging); + without_preview.preview = None; + run(&recorder, None, without_preview, record(2)); + assert_warning( + &recorder, + name, + diff, + values(None, None), + Some(ShadowMismatchLoggingOutcome::RenderError), + ); + assert_eq!( + recorder.logging_outcomes(), + vec![ShadowMismatchLoggingOutcome::RenderError] + ); + } + + // Without values to log, the differences need the hook. + let recorder = Arc::new(Recorder::default()); + let mut diff_only = operation( + "NoPreviewDiffOnly", + shadow(None, leaves(Some(false), Some(MismatchDiff::Paths))), + ); + diff_only.preview = None; + run(&recorder, None, diff_only, record(2)); + assert_warning( + &recorder, + "NoPreviewDiffOnly", + failed_diff(MismatchDiff::Paths), + None, + Some(ShadowMismatchLoggingOutcome::RenderError), + ); + + let recorder = Arc::new(Recorder::default()); + let mut hooked = operation("NoPreviewHook", shadow(Some(true), None)).mismatch_diff(|_, _| { + vec![MismatchEntry { + path: "/version".into(), + cached: None, + source: None, + }] + }); + hooked.preview = None; + run(&recorder, None, hooked, record(2)); + assert_warning( + &recorder, + "NoPreviewHook", + paths(MismatchDiff::Values, Some(r#"[{"path":"/version"}]"#)), + values(None, None), + Some(ShadowMismatchLoggingOutcome::RenderError), + ); +} + +#[test] +fn the_default_preview_keeps_the_json_rendering_through_clones_and_reassignment() { + let original = operation("DefaultPreview", shadow(Some(true), None)); + // The default preview renders the value's JSON when called directly. + let preview = original.preview.clone().expect("new installs a preview"); + assert_eq!(preview(&record(1)).as_deref(), Some(CACHED_JSON)); + let mut reassigned = original.clone(); + reassigned.preview = Some(preview); + for operation in [original.clone(), reassigned] { + let recorder = Arc::new(Recorder::default()); + run(&recorder, None, operation, record(2)); + assert_warning( + &recorder, + "DefaultPreview", + paths(MismatchDiff::Values, Some(DIFF_JSON)), + values(Some(CACHED_JSON), Some(SOURCE_JSON)), + None, + ); + } + + // Another operation's default preview is a preview set in place of this + // one's: it replaces the JSON, so its differences need a hook. + let recorder = Arc::new(Recorder::default()); + let mut borrowed = operation("DefaultPreview", shadow(Some(true), None)); + borrowed.preview = original.preview; + run(&recorder, None, borrowed, record(2)); + assert_warning( + &recorder, + "DefaultPreview", + failed_diff(MismatchDiff::Values), + values(Some(CACHED_JSON), Some(SOURCE_JSON)), + Some(ShadowMismatchLoggingOutcome::RenderError), + ); +} + +#[test] +fn an_invalid_runtime_leaf_keeps_the_valid_default_leaf_with_one_config_resolution() { + let recorder = confirmed( + "InvalidRuntimeLeaf", + shadow(None, leaves(Some(true), None)), + Some(json!({"shadow": {"mismatchLogging": {"diff": "everything"}}})), + ); + assert_eq!(recorder.config_resolutions(), 1); + assert_warning( + &recorder, + "InvalidRuntimeLeaf", + None, + values(Some(CACHED_JSON), Some(SOURCE_JSON)), + None, + ); +} + +#[test] +fn a_malformed_runtime_shorthand_owes_one_config_resolution_and_its_leaves_still_log() { + let recorder = confirmed( + "MalformedShorthand", + shadow(Some(true), None), + Some(json!({"shadow": { + "logMismatches": "yes", + "mismatchLogging": {"value": false, "diff": "paths"}, + }})), + ); + assert_eq!(recorder.config_resolutions(), 1); + assert_warning( + &recorder, + "MalformedShorthand", + paths(MismatchDiff::Paths, None), + None, + None, + ); + // A malformed shorthand alone turns logging off, still with one diagnostic. + let recorder = confirmed( + "MalformedShorthandAlone", + shadow(Some(true), None), + Some(json!({"shadow": {"logMismatches": null, "mismatchLogging": {"value": 1}}})), + ); + assert_eq!(recorder.config_resolutions(), 1); + assert!(recorder.warnings().is_empty()); + assert!(recorder.logging_outcomes().is_empty()); +} + +#[test] +fn the_recorded_table_rows_hold_end_to_end() { + for (name, policy, overlay, expected) in [ + ( + "RowShorthandTrueValueFalse", + shadow(Some(true), leaves(Some(false), None)), + None, + Some((paths(MismatchDiff::Values, Some(DIFF_JSON)), None)), + ), + ( + "RowShorthandFalseDiffPaths", + shadow(Some(false), leaves(None, Some(MismatchDiff::Paths))), + None, + Some((paths(MismatchDiff::Paths, None), None)), + ), + ( + "RowRuntimeFalseOverPaths", + shadow(None, leaves(None, Some(MismatchDiff::Paths))), + Some(json!({"shadow": {"logMismatches": false}})), + None, + ), + ( + "RowRuntimeTrueOverValueFalse", + shadow(None, leaves(Some(false), None)), + Some(json!({"shadow": {"logMismatches": true}})), + Some(( + paths(MismatchDiff::Values, Some(DIFF_JSON)), + values(Some(CACHED_JSON), Some(SOURCE_JSON)), + )), + ), + ] { + let recorder = confirmed(name, policy, overlay); + assert_eq!(recorder.config_resolutions(), 0, "{name}"); + match expected { + None => { + assert!(recorder.warnings().is_empty(), "{name}"); + assert!(recorder.logging_outcomes().is_empty(), "{name}"); + } + Some((diff, values)) => assert_warning(&recorder, name, diff, values, None), + } + } +} + +#[test] +fn the_disabled_overlay_turns_off_every_inherited_mismatch_logging_leaf() { + let all_on = shadow(Some(true), leaves(Some(true), Some(MismatchDiff::Values))); + let disabled = RuntimePolicy::from(Policy::disabled()).0; + let recorder = Arc::new(Recorder::default()); + let served = run( + &recorder, + Some(disabled.clone()), + operation("DisabledOverlay", all_on.clone()), + record(2), + ); + assert_eq!(*served, record(2)); + assert!(recorder.verdicts().is_empty()); + assert!(recorder.warnings().is_empty()); + + // With only the shadow ramp restored, the verdict runs, but the overlay's + // leaves replace both inherited ones. + let mut overlay = disabled; + overlay["shadow"]["ramp"] = json!(100); + overlay["ramp"] = json!({}); + let recorder = confirmed("DisabledLeaves", all_on, Some(overlay)); + assert!(recorder.warnings().is_empty()); + assert!(recorder.logging_outcomes().is_empty()); + assert_eq!(recorder.config_resolutions(), 0); +} + +type Gate = Arc>>>; + +/// A source whose first call waits for the gate, holding its shadow job. +fn gated_source( + gate: Gate, +) -> impl Fn(dialcache::Scope) -> Pin> + Send>> + + Send + + Sync + + 'static { + move |_| { + let held = gate.lock().take(); + Box::pin(async move { + if let Some(held) = held { + let _ = held.await; + } + Ok(record(2)) + }) + } +} + +#[test] +fn an_admitted_job_records_one_config_resolution_and_a_dropped_job_none() { + let recorder = Arc::new(Recorder::default()); + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = cache( + &executor, + &recorder, + Some(json!({"shadow": { + "logMismatches": 0, + "mismatchLogging": {"value": "yes", "diff": "everything"}, + }})), + ); + let (release, held) = oneshot::channel(); + let gate: Gate = Arc::new(Mutex::new(Some(held))); + for _ in 0..2 { + let cache = cache.clone(); + let source = gated_source(gate.clone()); + executor.block_on(async move { + let request = cache.enable_guard(); + let operation = operation("ConfigResolutionCount", shadow(Some(true), None)); + cache + .get_or_load(request.scope(), operation, source) + .await + .unwrap(); + }); + executor.drain(); + } + // The first job holds the only slot, so the second is dropped before admission. + assert_eq!(recorder.verdicts(), vec![ShadowOutcome::Dropped]); + assert_eq!(recorder.config_resolutions(), 1); + release.send(()).unwrap(); + executor.drain(); + assert_eq!( + recorder.verdicts(), + vec![ShadowOutcome::Dropped, ShadowOutcome::Mismatch] + ); + assert_eq!(recorder.config_resolutions(), 1); + assert!(recorder.warnings().is_empty()); +} diff --git a/rust/tests/shadow_write_deadline.rs b/rust/tests/shadow_write_deadline.rs index 33de6875..ac43b053 100644 --- a/rust/tests/shadow_write_deadline.rs +++ b/rust/tests/shadow_write_deadline.rs @@ -103,6 +103,7 @@ fn invoke(executor: &mut TestExecutor, cache: &DialCache) { .shadow(ShadowPolicy { ramp: Some(100.0), log_mismatches: None, + mismatch_logging: None, }), ) .budget(SourceBudget::Millis(5)), diff --git a/typescript/scripts/test-package.mjs b/typescript/scripts/test-package.mjs index 9b75d711..48ad6133 100644 --- a/typescript/scripts/test-package.mjs +++ b/typescript/scripts/test-package.mjs @@ -168,6 +168,12 @@ const rootConsumer = `import { type Serializer, type ShadowComparator, type ShadowConfig, + type ShadowMismatchDiff, + type ShadowMismatchDiffLevel, + type ShadowMismatchEntry, + type ShadowMismatchLoggingConfig, + type ShadowMismatchLoggingMetricLabels, + type ShadowMismatchLoggingOutcome, type ShadowValidationMetricLabels, type ShadowValidationOutcome, type StaleRecoveryMetricLabels, @@ -266,6 +272,10 @@ const shadowMetrics: DialCacheMetricsAdapter = { const outcome: ShadowValidationOutcome = labels.outcome; void outcome; }, + shadowMismatchLogging: (labels: ShadowMismatchLoggingMetricLabels) => { + const outcome: ShadowMismatchLoggingOutcome = labels.outcome; + void outcome; + }, }; const staleMetrics: DialCacheMetricsAdapter = { ...metrics, @@ -296,6 +306,27 @@ const shadowOutcomes: Readonly> = { dropped: true, }; void shadowOutcomes; +const shadowMismatchLoggingOutcomes: Readonly> = { + logged: true, + hook_error: true, + render_error: true, + logger_error: true, +}; +const shadowMismatchLoggingLabels: ShadowMismatchLoggingMetricLabels = { + cacheNamespace: "consumer-shadow-cache", + useCase: "Load", + keyType: "id", + outcome: "hook_error", +}; +// @ts-expect-error Mismatch-logging outcomes are a bounded metric category. +const unboundedShadowMismatchLoggingOutcome: ShadowMismatchLoggingOutcome = "skipped"; +const shadowMismatchDiffLevels: Readonly> = { + off: true, + paths: true, + values: true, +}; +// @ts-expect-error Mismatch diff levels are a bounded public choice. +const unboundedShadowMismatchDiffLevel: ShadowMismatchDiffLevel = "full"; const staleRecoveryOutcomes: Readonly> = { served: true, miss: true, @@ -334,11 +365,31 @@ const staleCacheConfig: DialCacheConfig = { shouldAttemptStaleRecovery: staleRecoveryPredicate, }; const staleCache = new DialCache(staleCacheConfig); +const shadowMismatchLogging: ShadowMismatchLoggingConfig = { value: false, diff: "paths" }; const shadowConfig: ShadowConfig = { ramp: 50, - logMismatches: true, + mismatchLogging: shadowMismatchLogging, }; const shadowKeyConfig = new DialCacheKeyConfig({ shadow: shadowConfig }); +// The shorthand stands for the whole group; leaves written beside it override it. +const shorthandShadowConfig: ShadowConfig = { + ramp: 50, + logMismatches: true, + mismatchLogging: { value: false }, +}; +const shorthandShadowKeyConfig = new DialCacheKeyConfig({ shadow: shorthandShadowConfig }); +const invalidShadowMismatchLogging: ShadowConfig = { + mismatchLogging: { + // @ts-expect-error The value leaf is a boolean, not a diff level. + value: "values", + }, +}; +const shadowMismatchEntry: ShadowMismatchEntry = { path: "/nested/scores/1", cached: 2 }; +const locationOnlyShadowMismatchEntry: ShadowMismatchEntry = { path: "/nickname" }; +// @ts-expect-error Every mismatch entry names its JSON Pointer. +const shadowMismatchEntryWithoutPath: ShadowMismatchEntry = { cached: 1, source: 2 }; +const stringShadowMismatchDiff: ShadowMismatchDiff = (cachedValue, sourceValue) => + cachedValue === sourceValue ? [] : [{ path: "", cached: cachedValue, source: sourceValue }]; const staleKeyConfig = new DialCacheKeyConfig({ ttlSec: { [CacheLayer.REMOTE]: 60 }, staleOnErrorMaxAgeSec: 300, @@ -356,6 +407,9 @@ const datadogOptions: DatadogMetricsOptions = { }; const datadogMetrics = createDatadogDialCacheMetrics(datadogOptions); const datadogClassAdapter = new DatadogDialCacheMetrics(datadogOptions); +const datadogCountsShadowMismatchLogging: "shadowMismatchLogging" extends keyof DatadogDialCacheMetrics + ? true + : false = true; // @ts-expect-error The observation type is an explicit, required choice. const missingObservationType: DatadogMetricsOptions = { client: dogStatsDClient }; const cache = new DialCache({ namespace: "consumer-cache", metrics }); @@ -426,6 +480,7 @@ const load = cache.cached(async (id: string) => id, { cacheKey: (id) => id, fallbackTimeoutMs: 1_000, shadowComparator: stringShadowComparator, + shadowMismatchDiff: stringShadowMismatchDiff, shouldAttemptStaleRecovery: staleRecoveryPredicate, defaultConfig: new DialCacheKeyConfig({ ttlSec: { [CacheLayer.LOCAL]: 60, [CacheLayer.REMOTE]: 60 }, @@ -448,6 +503,8 @@ const inlineAsync: Promise<{ readonly id: string }> = cache.getOrLoad( { ...inlineOptionsFor("InlineAsync"), shadowComparator: (cachedValue, sourceValue) => cachedValue.id === sourceValue.id, + shadowMismatchDiff: (cachedValue, sourceValue) => + cachedValue.id === sourceValue.id ? [] : [{ path: "/id", cached: cachedValue.id, source: sourceValue.id }], shouldAttemptStaleRecovery: staleRecoveryPredicate, }, ); @@ -536,6 +593,21 @@ cache.cached(async (id: string) => id, { // @ts-expect-error Shadow comparators must return a boolean synchronously. shadowComparator: async (cachedValue, sourceValue) => cachedValue === sourceValue, }); +cache.cached(async (id: string) => id, { + ...optionsFor("InvalidAsyncShadowMismatchDiff"), + // @ts-expect-error Shadow mismatch diff hooks must return their entries synchronously. + shadowMismatchDiff: async (cachedValue, sourceValue) => [{ path: "", cached: cachedValue, source: sourceValue }], +}); +cache.getOrLoad(async () => "inline", { + ...inlineOptionsFor("InvalidInlineAsyncShadowMismatchDiff"), + // @ts-expect-error Inline shadow mismatch diff hooks must also return their entries synchronously. + shadowMismatchDiff: async () => [], +}); +cache.cached(async (id: string) => id, { + ...optionsFor("InvalidShadowMismatchDiffValueType"), + // @ts-expect-error Shadow mismatch diff hooks receive the cached function's resolved value type. + shadowMismatchDiff: (cachedValue: number, sourceValue: number) => [{ path: "", cached: cachedValue, source: sourceValue }], +}); const requestLocalConfig = new DialCacheKeyConfig({ requestLocal: true }); const uncoalescedConfig = new DialCacheKeyConfig({ coalesce: false }); @@ -776,6 +848,16 @@ void coalesceFlag; void structuralConfigProvider; void shadowCache; void shadowKeyConfig; +void shorthandShadowKeyConfig; +void invalidShadowMismatchLogging; +void shadowMismatchLoggingOutcomes; +void shadowMismatchLoggingLabels; +void unboundedShadowMismatchLoggingOutcome; +void shadowMismatchDiffLevels; +void unboundedShadowMismatchDiffLevel; +void shadowMismatchEntry; +void locationOnlyShadowMismatchEntry; +void shadowMismatchEntryWithoutPath; void staleMetrics; void staleCache; void staleCacheConfig; @@ -880,6 +962,7 @@ void rootHasNoDeterministicRampSampler; void rootHasNoRandomRampSampler; void datadogMetrics; void datadogClassAdapter; +void datadogCountsShadowMismatchLogging; void missingObservationType; void invalidAsyncStaleRecoveryConfig; `; @@ -1255,7 +1338,9 @@ if ( || esmDisabledOverlay.coalesce !== undefined || esmDisabledOverlay.staleOnErrorMaxAgeSec !== 0 || esmDisabledOverlay.shadow?.ramp !== 0 - || esmDisabledOverlay.shadow.logMismatches !== false + || esmDisabledOverlay.shadow.logMismatches !== undefined + || esmDisabledOverlay.shadow.mismatchLogging?.value !== false + || esmDisabledOverlay.shadow.mismatchLogging.diff !== "off" || esmDisabledOverlay.ramp[root.CacheLayer.LOCAL] !== 0 || esmDisabledOverlay.ramp[root.CacheLayer.REMOTE] !== 0 ) { @@ -1709,7 +1794,9 @@ if ( || cjsDisabledOverlay.coalesce !== undefined || cjsDisabledOverlay.staleOnErrorMaxAgeSec !== 0 || cjsDisabledOverlay.shadow?.ramp !== 0 - || cjsDisabledOverlay.shadow.logMismatches !== false + || cjsDisabledOverlay.shadow.logMismatches !== undefined + || cjsDisabledOverlay.shadow.mismatchLogging?.value !== false + || cjsDisabledOverlay.shadow.mismatchLogging.diff !== "off" || cjsDisabledOverlay.ramp[root.CacheLayer.LOCAL] !== 0 || cjsDisabledOverlay.ramp[root.CacheLayer.REMOTE] !== 0 ) { diff --git a/typescript/src/config.ts b/typescript/src/config.ts index 0a2d2643..5422da25 100644 --- a/typescript/src/config.ts +++ b/typescript/src/config.ts @@ -13,15 +13,50 @@ export type LayerConfig = Partial>; /** Synchronously classifies whether one source rejection may use a retained Redis candidate. */ export type StaleRecoveryPredicate = (error: unknown) => boolean; +/** + * How much of the structural difference a confirmed-mismatch warning logs: + * `"off"` nothing, `"paths"` the JSON Pointers of the differing fields, and + * `"values"` those pointers with the cached and source values at each one. + */ +export type ShadowMismatchDiffLevel = "off" | "paths" | "values"; + +/** + * Content of the one warning emitted per confirmed shadow mismatch. Both + * leaves default off, and the warning is emitted only when `value` is true or + * `diff` is above `"off"`. Leaves merge independently at runtime, like + * cache-layer leaves. + */ +export interface ShadowMismatchLoggingConfig { + /** Include bounded native-JSON strings for both compared values. */ + readonly value?: boolean; + /** Include the differing fields at this level. */ + readonly diff?: ShadowMismatchDiffLevel; +} + /** Per-use-case runtime policy for detached Redis shadow work. */ export interface ShadowConfig { /** Independent stable cohort percentage. Omitted and zero disable shadow work. */ readonly ramp?: number; /** - * Emit one warning with the logical key and bounded native-JSON strings for - * the compared values for each confirmed mismatch. Defaults to false. + * Shorthand for the whole `mismatchLogging` group: `true` stands for + * `{ value: true, diff: "values" }` and `false` for + * `{ value: false, diff: "off" }`. Leaves written in this same object's + * `mismatchLogging` override the expanded values before runtime overlays + * merge. */ readonly logMismatches?: boolean; + /** Confirmed-mismatch warning content. Omitted leaves default off. */ + readonly mismatchLogging?: ShadowMismatchLoggingConfig; +} + +interface DialCacheKeyConfigInput { + readonly ttlSec?: LayerConfig; + readonly ramp?: LayerConfig; + readonly shadow?: ShadowConfig; + readonly requestLocal?: boolean; + readonly coalesce?: boolean; + readonly staleOnErrorMaxAgeSec?: number; + readonly remoteReadTimeoutMs?: number; } export class DialCacheKeyConfig { @@ -59,15 +94,7 @@ export class DialCacheKeyConfig { */ readonly remoteReadTimeoutMs?: number; - constructor(config: { - ttlSec?: LayerConfig; - ramp?: LayerConfig; - shadow?: ShadowConfig; - requestLocal?: boolean; - coalesce?: boolean; - staleOnErrorMaxAgeSec?: number; - remoteReadTimeoutMs?: number; - }) { + constructor(config: DialCacheKeyConfigInput) { if (config === null || typeof config !== "object" || Array.isArray(config)) { throw new TypeError("DialCache key config must be an object"); } @@ -129,7 +156,13 @@ export class DialCacheKeyConfig { staleOnErrorMaxAgeSec: 0, shadow: { ramp: 0, - logMismatches: false, + // `Required` keeps this overlay exhaustive: leaves merge + // independently, so an omitted leaf would let an inherited one + // survive disabled(). + mismatchLogging: { + value: false, + diff: "off", + } satisfies Required, }, ramp: { [CacheLayer.LOCAL]: 0, @@ -156,6 +189,19 @@ function cloneShadowConfig(config: ShadowConfig | undefined): ShadowConfig | und if (config === null || typeof config !== "object" || Array.isArray(config)) { throw new TypeError("DialCache shadow config must be an object"); } + const clone: { -readonly [Field in keyof ShadowConfig]: ShadowConfig[Field] } = { ...config }; + if (clone.mismatchLogging !== undefined) { + clone.mismatchLogging = cloneMismatchLoggingConfig(clone.mismatchLogging); + } + return clone; +} + +function cloneMismatchLoggingConfig(config: ShadowMismatchLoggingConfig): ShadowMismatchLoggingConfig { + if (config === null || typeof config !== "object" || Array.isArray(config)) { + throw new TypeError("DialCache shadow mismatchLogging config must be an object"); + } + // Leaf values stay unvalidated, like layer-map leaves, so a malformed + // runtime leaf fails open narrowly at resolution. return { ...config }; } diff --git a/typescript/src/datadog.ts b/typescript/src/datadog.ts index ea0f2887..d9435017 100644 --- a/typescript/src/datadog.ts +++ b/typescript/src/datadog.ts @@ -9,6 +9,7 @@ import type { InvalidationMetricLabels, MissMetricLabels, SerializationMetricLabels, + ShadowMismatchLoggingMetricLabels, ShadowValidationMetricLabels, StaleRecoveryMetricLabels, } from "./metrics.js"; @@ -36,7 +37,10 @@ export interface DatadogMetricsOptions { type CacheTag = "cache_namespace" | "use_case" | "key_type" | "layer"; type DatadogTags = Record; -type OutcomeMetricLabels = ShadowValidationMetricLabels | StaleRecoveryMetricLabels; +type OutcomeMetricLabels = + | ShadowValidationMetricLabels + | ShadowMismatchLoggingMetricLabels + | StaleRecoveryMetricLabels; type Observation = (name: string, value: number, tags: DatadogTags) => void; const DEFAULT_NAMESPACE = "dialcache"; @@ -52,6 +56,7 @@ const METRIC_SUFFIXES = { coalesced: "coalesced.count", shadowValidation: "shadow.count", shadowValueAge: "shadow.value_age", + shadowMismatchLogging: "shadow_mismatch_logging.count", futureTimestampOffset: "future_timestamp_offset", staleRecovery: "stale_recovery.count", staleRecoveryValueAge: "stale_recovery.value_age", @@ -134,6 +139,10 @@ export class DatadogDialCacheMetrics implements DialCacheMetricsAdapter { this.observe(this.metricNames.shadowValueAge, seconds, outcomeTags(labels)); } + shadowMismatchLogging(labels: ShadowMismatchLoggingMetricLabels): void { + return this.increment(this.metricNames.shadowMismatchLogging, outcomeTags(labels)); + } + observeFutureTimestampOffset(labels: CacheMetricLabels, seconds: number): void { this.observe(this.metricNames.futureTimestampOffset, seconds, cacheTags(labels)); } diff --git a/typescript/src/dialcache.ts b/typescript/src/dialcache.ts index 79e940ab..dfbb2fcc 100644 --- a/typescript/src/dialcache.ts +++ b/typescript/src/dialcache.ts @@ -8,6 +8,7 @@ import { type CacheConfigProvider, type DialCacheConfig, type Logger, + type ShadowConfig, type StaleRecoveryPredicate, } from "./config.js"; import { DialCacheContext, getOrCreateRequestLocalCache, type RequestLocalCache } from "./context.js"; @@ -22,6 +23,7 @@ import { type DisabledReason, type MetricErrorKind, type MetricLayer, + type ShadowMismatchLoggingOutcome, type ShadowValidationOutcome, } from "./metrics.js"; import { @@ -43,12 +45,21 @@ import { deterministicShadowRampSample } from "./internal/ramp.js"; import { RedisCache, type FutureFramePolicy } from "./internal/redis-cache.js"; import { fetchKeyConfig, + isShadowMismatchDiffLevel, + resolveMismatchLogging, resolveRemoteLayerConfigResult, type LayerConfigResolution, type ResolvedLayerConfig, type ResolvedRemoteLayerConfig, } from "./internal/runtime-config.js"; -import { shadowMismatchLogDetails } from "./internal/shadow-log-json.js"; +import { + failedShadowMismatchLogFields, + previewShadowLogKey, + renderShadowMismatchLog, + type ShadowMismatchLogContent, + type ShadowMismatchLogPlan, +} from "./internal/shadow-log-json.js"; +import { fulfills, isPromiseLike } from "./internal/thenable.js"; type CacheKeyArgs = Record; type Id = string | number | bigint; @@ -67,6 +78,29 @@ export type CachedValue = Awaited>; * side-effect-free, non-mutating, and bounded. */ export type ShadowComparator = (cachedValue: Value, sourceValue: Value) => boolean; +/** + * One differing field of a confirmed shadow mismatch: an RFC 6901 JSON Pointer + * into the compared value and, optionally, the value on each side there. Omit + * a side the field does not have, or both when only the location is known. + */ +export interface ShadowMismatchEntry { + readonly path: string; + readonly cached?: unknown; + readonly source?: unknown; +} +/** + * Replaces the built-in JSON difference computation for confirmed-mismatch + * warnings, which runs only when `shadow.mismatchLogging.diff` is above + * `"off"`. Inputs are borrowed snapshots. Like comparators, hooks must be + * synchronous, deterministic, side-effect-free, non-mutating, and bounded. + * A throw, a promise-like or a non-array logs `null` for the paths and the + * diff; DialCache never falls back to the built-in computation. Entries are + * logged as the hook returns them. + */ +export type ShadowMismatchDiff = ( + cachedValue: Value, + sourceValue: Value, +) => readonly ShadowMismatchEntry[]; type CacheKeySelector = (...args: Parameters) => CacheKeySpec; type IsAny = 0 extends 1 & T ? true : false; @@ -133,6 +167,13 @@ interface CacheOperationOptionsBase { * This is stable use-case behavior, not runtime rollout configuration. */ readonly shadowComparator?: ShadowComparator; + /** + * Replaces the built-in JSON difference computation behind + * `mismatchPaths` and `diffJson` in confirmed-mismatch warnings. Must be a + * synchronous function. This is stable use-case behavior, not runtime + * rollout configuration. + */ + readonly shadowMismatchDiff?: ShadowMismatchDiff; /** * Overrides the DialCache-instance source-error classifier for this use * case, replacing both the instance and built-in policies. Must be @@ -238,13 +279,16 @@ interface ShadowFlight { interface ShadowValidationPlan { readonly source: () => Promise; readonly comparator: ShadowComparator; + readonly mismatchDiff: ShadowMismatchDiff | undefined; readonly timeoutMs: number; readonly didCallerFallbackTimeout: () => boolean; } interface ShadowMismatchDetails { + readonly logPlan: ShadowMismatchLogPlan; readonly cachedValue: unknown; readonly sourceValue: unknown; + readonly mismatchDiff: ShadowMismatchDiff | undefined; } type ShadowValidationStart = @@ -274,6 +318,7 @@ const defaultConfigProvider: CacheConfigProvider = () => null; const defaultLogger: Logger = console; const defaultStaleRecoveryPredicate: StaleRecoveryPredicate = (error) => error instanceof FallbackTimeoutError; +const SHADOW_MISMATCH_WARNING = "DialCache shadow validation mismatch"; export class DialCache { private readonly context = new DialCacheContext(); @@ -281,7 +326,7 @@ export class DialCache { private readonly useCases = new Set(); private readonly configProvider: CacheConfigProvider; private readonly namespace: string; - private readonly logger: Logger; + private readonly logger: SafeLogger; private readonly redisCache: RedisCache | null; private readonly metrics: DialCacheMetricsAdapter | null; private readonly staleRecoveryPredicate: StaleRecoveryPredicate; @@ -375,6 +420,7 @@ export class DialCache { const defaultConfig = snapshotDefaultConfig(options.defaultConfig); const fallbackTimeoutMs = resolveFallbackTimeoutMs(options.fallbackTimeoutMs); const shadowComparator = resolveShadowComparator(options.shadowComparator); + const shadowMismatchDiff = resolveShadowMismatchDiff(options.shadowMismatchDiff); const staleRecoveryPredicate = resolveStaleRecoveryPredicate( options.shouldAttemptStaleRecovery, this.staleRecoveryPredicate, @@ -391,6 +437,7 @@ export class DialCache { defaultConfig, fallbackTimeoutMs, shadowComparator, + shadowMismatchDiff, staleRecoveryPredicate, ); } @@ -405,6 +452,7 @@ export class DialCache { const defaultConfig = snapshotDefaultConfig(options.defaultConfig); const fallbackTimeoutMs = resolveFallbackTimeoutMs(options.fallbackTimeoutMs); const shadowComparator = resolveShadowComparator(options.shadowComparator); + const shadowMismatchDiff = resolveShadowMismatchDiff(options.shadowMismatchDiff); const staleRecoveryPredicate = resolveStaleRecoveryPredicate( options.shouldAttemptStaleRecovery, this.staleRecoveryPredicate, @@ -418,6 +466,7 @@ export class DialCache { defaultConfig, fallbackTimeoutMs, shadowComparator, + shadowMismatchDiff, staleRecoveryPredicate, ); } @@ -429,6 +478,7 @@ export class DialCache { defaultConfig: DialCacheKeyConfig | null, fallbackTimeoutMs: number | null, shadowComparator: ShadowComparator, + shadowMismatchDiff: ShadowMismatchDiff | undefined, staleRecoveryPredicate: StaleRecoveryPredicate, ): Promise { const rawFallback = async (): Promise => await load(); @@ -456,6 +506,7 @@ export class DialCache { const shadowValidation: ShadowValidationPlan = { source: rawFallback, comparator: shadowComparator, + mismatchDiff: shadowMismatchDiff, timeoutMs: fallbackTimeoutMs ?? DEFAULT_FALLBACK_TIMEOUT_MS, didCallerFallbackTimeout: () => callerFallbackTimedOut, }; @@ -902,7 +953,7 @@ export class DialCache { // The public contract is synchronous. Consume an accidental rejecting // thenable without awaiting it, and fail closed without delaying the // original source rejection. - void settleUnexpectedThenable(result); + void fulfills(result); this.logger.warn("DialCache stale recovery predicate returned a non-boolean; recovery was denied"); return false; } @@ -929,7 +980,7 @@ export class DialCache { return; } - const resolvedShadowConfig = shadowConfig as Record; + const resolvedShadowConfig = shadowConfig as ShadowConfig; const shadowPercentage: unknown = resolvedShadowConfig.ramp; if (shadowPercentage === undefined || shadowPercentage === 0) { return; @@ -954,7 +1005,7 @@ export class DialCache { this.recordShadowValidation(key, "dropped"); return; } - const logMismatches = this.resolveShadowLogging(key, resolvedShadowConfig); + const logPlan = this.resolveShadowLogging(key, resolvedShadowConfig); const flight: ShadowFlight = { cachedFrame: start.kind === "retained" ? start.frame : null, @@ -974,23 +1025,20 @@ export class DialCache { runStart, validation, readTimeoutMs, - logMismatches, + logPlan, ); } + /** The leaves captured at admission; `null` when both are off. */ private resolveShadowLogging( key: DialCacheKey, - shadowConfig: Record, - ): boolean { - const configuredLogMismatches = shadowConfig.logMismatches; - if (configuredLogMismatches === undefined) { - return false; - } - if (typeof configuredLogMismatches !== "boolean") { + shadowConfig: ShadowConfig, + ): ShadowMismatchLogPlan | null { + const { value, diff, invalid } = resolveMismatchLogging(shadowConfig); + if (invalid) { this.recordError(key, CacheLayer.REMOTE, "config_resolution"); - return false; } - return configuredLogMismatches; + return value || diff !== "off" ? { value, diff } : null; } private deferShadowValidation( @@ -1000,7 +1048,7 @@ export class DialCache { start: ShadowValidationRunStart, validation: ShadowValidationPlan, readTimeoutMs: number, - logMismatches: boolean, + logPlan: ShadowMismatchLogPlan | null, ): void { setImmediate(() => { this.runShadowValidation( @@ -1010,7 +1058,7 @@ export class DialCache { start, validation, readTimeoutMs, - logMismatches, + logPlan, ); }).unref(); } @@ -1022,7 +1070,7 @@ export class DialCache { start: ShadowValidationRunStart, plan: ShadowValidationPlan, readTimeoutMs: number, - logMismatches: boolean, + logPlan: ShadowMismatchLogPlan | null, ): void { const pendingRedisReads = new Set>(); let operationFinished = false; @@ -1184,7 +1232,7 @@ export class DialCache { try { const result: unknown = plan.comparator(cachedValue, sourceValue); if (typeof result !== "boolean") { - await settleUnexpectedThenable(result); + await fulfills(result); if (abandonIfExpired()) { return "timeout"; } @@ -1222,8 +1270,8 @@ export class DialCache { if (confirmationFrame === null || !redisPayloadsEqual(originalFrame.payload, confirmationFrame.payload)) { return "superseded"; } - if (logMismatches) { - mismatchDetails = { cachedValue, sourceValue }; + if (logPlan !== null) { + mismatchDetails = { logPlan, cachedValue, sourceValue, mismatchDiff: plan.mismatchDiff }; } validatedValueAgeSeconds = shadowValueAgeSeconds(originalFrame.createdAtMs); return "mismatch"; @@ -1234,7 +1282,7 @@ export class DialCache { }); void validation.then( - (outcome) => this.recordShadowValidation(key, outcome, logMismatches, mismatchDetails, validatedValueAgeSeconds), + (outcome) => this.recordShadowValidation(key, outcome, mismatchDetails, validatedValueAgeSeconds), () => this.recordShadowValidation(key, "timeout"), ); } @@ -1242,7 +1290,6 @@ export class DialCache { private recordShadowValidation( key: DialCacheKey, outcome: ShadowValidationOutcome, - logMismatches = false, mismatchDetails?: ShadowMismatchDetails, valueAgeSeconds?: number, ): void { @@ -1256,35 +1303,66 @@ export class DialCache { if (valueAgeSeconds !== undefined) { this.metrics?.observeShadowValueAge?.(labels, valueAgeSeconds); } - if (outcome !== "mismatch" || !logMismatches) { - return; + // Only a confirmed mismatch with logging on carries details. + if (mismatchDetails !== undefined) { + this.warnShadowMismatch(key, mismatchDetails, valueAgeSeconds); } + } - const warning = { + /** + * Emits the confirmed-mismatch warning and its one `shadowMismatchLogging` + * event. Runs after the verdict was recorded and never changes it; any + * construction failure logs less, never more, and nothing reaches the caller. + */ + private warnShadowMismatch( + key: DialCacheKey, + details: ShadowMismatchDetails, + valueAgeSeconds: number | undefined, + ): void { + const metadata = { cacheNamespace: key.namespace, useCase: key.useCase, keyType: key.keyType, outcome: "mismatch", + cachedValueAgeSeconds: valueAgeSeconds ?? null, } as const; - if (mismatchDetails !== undefined) { - try { - this.logger.warn( - "DialCache shadow validation mismatch", - { - ...warning, - ...shadowMismatchLogDetails( - key.urn, - mismatchDetails.cachedValue, - mismatchDetails.sourceValue, - ), - }, - ); - return; - } catch { - // JSON detail construction is best-effort; preserve the metadata warning. - } + let cacheKey: string | undefined; + let content: ShadowMismatchLogContent; + try { + cacheKey = previewShadowLogKey(key.urn); + content = renderShadowMismatchLog( + details.logPlan, + details.cachedValue, + details.sourceValue, + details.mismatchDiff, + ); + } catch { + // Last resort: nothing may escape into the detached verdict handler. + content = { fields: failedShadowMismatchLogFields(details.logPlan), failure: "render_error" }; + } + const failure = content.failure; + const delivered = this.logger.warn(SHADOW_MISMATCH_WARNING, { + ...metadata, + ...(cacheKey === undefined ? {} : { cacheKey }), + ...content.fields, + ...(failure === null ? {} : { loggingOutcome: failure }), + }); + const recordOutcome = (outcome: ShadowMismatchLoggingOutcome): void => { + this.metrics?.shadowMismatchLogging?.({ + cacheNamespace: key.namespace, + useCase: key.useCase, + keyType: key.keyType, + outcome, + }); + }; + if (failure !== null) { + recordOutcome(failure); + } else if (typeof delivered === "boolean") { + recordOutcome(delivered ? "logged" : "logger_error"); + } else { + // An asynchronous logger settles the outcome when its promise does. + void delivered.then((loggerSucceeded) => recordOutcome(loggerSucceeded ? "logged" : "logger_error")); } - this.logger.warn("DialCache shadow validation mismatch", warning); } private async resolveLocalLayerConfig( @@ -1609,11 +1687,21 @@ function snapshotDefaultConfig(config: DialCacheKeyConfig | null | undefined): D if (snapshot.shadow.logMismatches !== undefined && typeof snapshot.shadow.logMismatches !== "boolean") { throw new TypeError("DialCache defaultConfig shadow.logMismatches must be a boolean"); } + const mismatchLogging = snapshot.shadow.mismatchLogging; + if (mismatchLogging?.value !== undefined && typeof mismatchLogging.value !== "boolean") { + throw new TypeError("DialCache defaultConfig shadow.mismatchLogging.value must be a boolean"); + } + if (mismatchLogging?.diff !== undefined && !isShadowMismatchDiffLevel(mismatchLogging.diff)) { + throw new TypeError('DialCache defaultConfig shadow.mismatchLogging.diff must be "off", "paths" or "values"'); + } } Object.freeze(snapshot.ttlSec); Object.freeze(snapshot.ramp); if (snapshot.shadow !== undefined) { + if (snapshot.shadow.mismatchLogging !== undefined) { + Object.freeze(snapshot.shadow.mismatchLogging); + } Object.freeze(snapshot.shadow); } return Object.freeze(snapshot); @@ -1668,7 +1756,12 @@ function withFallbackTimeout( }); } -function safeLogger(logger: Logger): Logger { +/** The application logger; each method reports whether it returned without throwing. */ +type SafeLogger = { + readonly [Method in keyof Logger]: (...args: Parameters) => boolean | Promise; +}; + +function safeLogger(logger: Logger): SafeLogger { return { debug: (...args: Parameters) => callObserver(() => logger.debug(...args)), error: (...args: Parameters) => callObserver(() => logger.error(...args)), @@ -1700,6 +1793,7 @@ function safeMetrics(metrics: DialCacheMetricsAdapter | null): DialCacheMetricsA callObserver(() => metrics.observeStaleRecoveryValueAge?.(labels, seconds)), observeShadowValueAge: (labels, seconds) => callObserver(() => metrics.observeShadowValueAge?.(labels, seconds)), + shadowMismatchLogging: (labels) => callObserver(() => metrics.shadowMismatchLogging?.(labels)), observeFutureTimestampOffset: (labels, seconds) => callObserver(() => metrics.observeFutureTimestampOffset?.(labels, seconds)), observeGet: (labels, seconds) => callObserver(() => metrics.observeGet(labels, seconds)), @@ -1712,16 +1806,18 @@ function safeMetrics(metrics: DialCacheMetricsAdapter | null): DialCacheMetricsA }; } -function callObserver(record: () => unknown): void { +/** + * Runs an injected observer in isolation and reports whether it succeeded: + * `false` when it throws, and for a returned promise-like, a promise of whether + * it fulfilled. Observer failures never produce unhandled rejections. + */ +function callObserver(record: () => unknown): boolean | Promise { try { const result = record(); - if (result !== undefined) { - void Promise.resolve(result).catch(() => { - // Observer failures must not produce unhandled rejections. - }); - } + return isPromiseLike(result) ? fulfills(result) : true; } catch { // Injected observers must not affect cache correctness or application fallbacks. + return false; } } @@ -1731,6 +1827,15 @@ function resolveShadowComparator( return comparator ?? isDeepStrictEqual; } +function resolveShadowMismatchDiff( + hook: ShadowMismatchDiff | undefined, +): ShadowMismatchDiff | undefined { + if (hook !== undefined && typeof hook !== "function") { + throw new TypeError("DialCache shadowMismatchDiff must be a function"); + } + return hook; +} + function resolveStaleRecoveryPredicate( predicate: StaleRecoveryPredicate | undefined, fallback: StaleRecoveryPredicate, @@ -1770,15 +1875,3 @@ function redisPayloadsEqual(left: RedisCachePayload, right: RedisCachePayload): } return Buffer.isBuffer(right) && right.equals(Buffer.from(left, "utf8")); } - -async function settleUnexpectedThenable(value: unknown): Promise { - if (value === null || (typeof value !== "object" && typeof value !== "function")) { - return; - } - try { - await Promise.resolve(value); - } catch { - // Synchronous extension points may accidentally return a rejected thenable; - // consume it without letting that rejection affect cache control flow. - } -} diff --git a/typescript/src/index.ts b/typescript/src/index.ts index 8eac0926..145b501e 100644 --- a/typescript/src/index.ts +++ b/typescript/src/index.ts @@ -5,6 +5,8 @@ export type { LayerConfig, Logger, ShadowConfig, + ShadowMismatchDiffLevel, + ShadowMismatchLoggingConfig, StaleRecoveryPredicate, } from "./config.js"; export { DialCacheContext } from "./context.js"; @@ -25,6 +27,8 @@ export type { MetricLayer, MissMetricLabels, SerializationMetricLabels, + ShadowMismatchLoggingMetricLabels, + ShadowMismatchLoggingOutcome, ShadowValidationMetricLabels, ShadowValidationOutcome, StaleRecoveryMetricLabels, @@ -47,6 +51,8 @@ export type { GetOrLoadOptions, ProcessCoalescingState, ShadowComparator, + ShadowMismatchDiff, + ShadowMismatchEntry, } from "./dialcache.js"; export { DialCacheKey, invalidationPrefix, normalizeArgs, redisClusterHashTag } from "./key.js"; export type { DialCacheKeyInit } from "./key.js"; diff --git a/typescript/src/internal/runtime-config.ts b/typescript/src/internal/runtime-config.ts index 9f10f68f..12414249 100644 --- a/typescript/src/internal/runtime-config.ts +++ b/typescript/src/internal/runtime-config.ts @@ -4,6 +4,7 @@ import { type CacheConfigProvider, type LayerConfig, type ShadowConfig, + type ShadowMismatchDiffLevel, } from "../config.js"; import type { DialCacheKey } from "../key.js"; import type { DisabledReason } from "../metrics.js"; @@ -146,6 +147,103 @@ export function resolveRemoteLayerConfigResult( }; } +/** + * Leaves one shadow group writes, before validation; `undefined` is unwritten. + * `invalid` marks a malformed shorthand, which still owes one config_resolution. + */ +export interface MismatchLoggingLeaves { + readonly value?: unknown; + readonly diff?: unknown; + readonly invalid?: true; +} + +/** The effective mismatch-logging leaves and whether any written input was invalid. */ +export interface MismatchLoggingResolution { + readonly value: boolean; + readonly diff: ShadowMismatchDiffLevel; + readonly invalid: boolean; +} + +// `satisfies Record<…, true>` fails to compile when the level union gains a +// member this set lacks. +const DIFF_LEVELS = { + off: true, + paths: true, + values: true, +} as const satisfies Record; + +export function isShadowMismatchDiffLevel(value: unknown): value is ShadowMismatchDiffLevel { + return typeof value === "string" && Object.hasOwn(DIFF_LEVELS, value); +} + +/** + * Expands one shadow group's `logMismatches` shorthand into the leaves it + * stands for, `true` as `{ value: true, diff: "values" }` and `false` as + * `{ value: false, diff: "off" }`, then lets the leaves written in the same + * group's `mismatchLogging` override them. A malformed shorthand expands to + * both leaves off and is marked invalid. + */ +export function expandMismatchLogging(shadow: ShadowConfig): MismatchLoggingLeaves { + // Runtime leaves stay unvalidated until resolution, so they are read as unknown. + const shorthand: unknown = shadow.logMismatches; + const expanded: MismatchLoggingLeaves = shorthand === undefined + ? {} + : shorthand === true + ? { value: true, diff: "values" } + : shorthand === false + ? { value: false, diff: "off" } + : { value: false, diff: "off", invalid: true }; + const written: { readonly value?: unknown; readonly diff?: unknown } | undefined = shadow.mismatchLogging; + if (written === undefined) { + return expanded; + } + const leaves: { value?: unknown; diff?: unknown; invalid?: true } = {}; + const value = written.value !== undefined ? written.value : expanded.value; + const diff = written.diff !== undefined ? written.diff : expanded.diff; + if (value !== undefined) { + leaves.value = value; + } + if (diff !== undefined) { + leaves.diff = diff; + } + if (expanded.invalid === true) { + leaves.invalid = true; + } + return leaves; +} + +/** + * Resolves the effective mismatch-logging leaves of one shadow group. Each + * leaf validates independently: an omitted leaf is off, an invalid leaf acts + * off, and `invalid` reports that the group needs one config_resolution. + */ +export function resolveMismatchLogging(shadow: ShadowConfig): MismatchLoggingResolution { + const leaves = expandMismatchLogging(shadow); + const value = resolveValueLeaf(leaves.value); + const diff = resolveDiffLeaf(leaves.diff); + return { + value: value ?? false, + diff: diff ?? "off", + invalid: leaves.invalid === true || value === null || diff === null, + }; +} + +/** `null` marks an invalid leaf. */ +function resolveValueLeaf(leaf: unknown): boolean | null { + if (leaf === undefined) { + return false; + } + return typeof leaf === "boolean" ? leaf : null; +} + +/** `null` marks an invalid leaf. */ +function resolveDiffLeaf(leaf: unknown): ShadowMismatchDiffLevel | null { + if (leaf === undefined) { + return "off"; + } + return isShadowMismatchDiffLevel(leaf) ? leaf : null; +} + function mergeKeyConfig( defaultConfig: DialCacheKeyConfig | null, runtimeConfig: DialCacheKeyConfig | null | undefined, @@ -170,7 +268,10 @@ function mergeKeyConfig( const staleOnErrorMaxAgeSec = overlay?.staleOnErrorMaxAgeSec !== undefined ? overlay.staleOnErrorMaxAgeSec : defaultConfig?.staleOnErrorMaxAgeSec; - const shadow = mergeShadowConfig(defaultConfig?.shadow, overlay?.shadow); + const shadow = mergeShadowConfig( + defaultConfig?.shadow, + overlay?.shadow, + ); return new DialCacheKeyConfig({ ttlSec: mergeLayerConfig(defaultConfig?.ttlSec, overlay?.ttlSec, "ttlSec"), @@ -228,19 +329,47 @@ function mergeShadowConfig( return undefined; } + const merged: { ramp?: unknown; logMismatches?: unknown; mismatchLogging?: { value?: unknown; diff?: unknown } } = {}; const ramp = overlay?.ramp !== undefined ? overlay.ramp : defaults?.ramp; - const logMismatches = overlay?.logMismatches !== undefined - ? overlay.logMismatches - : defaults?.logMismatches; - - return { - ...(ramp === undefined ? {} : { ramp }), - ...(logMismatches === undefined ? {} : { logMismatches }), - }; + if (ramp !== undefined) { + merged.ramp = ramp; + } + // Each layer's shorthand expands before the leaf-wise merge, so a runtime + // `logMismatches` replaces both inherited leaves while a leaf written beside + // it still wins within its own layer. + const inherited = defaults === undefined ? {} : expandMismatchLogging(defaults); + const written = overlay === undefined ? {} : expandMismatchLogging(overlay); + // Only the runtime layer can carry a malformed shorthand: registration + // rejects one in a static default. It stays raw, so admission records its + // config_resolution; its expanded leaves already won the merge. + if (written.invalid === true) { + merged.logMismatches = overlay?.logMismatches; + } + const value = written.value !== undefined ? written.value : inherited.value; + const diff = written.diff !== undefined ? written.diff : inherited.diff; + if (value !== undefined || diff !== undefined) { + const group: { value?: unknown; diff?: unknown } = {}; + if (value !== undefined) { + group.value = value; + } + if (diff !== undefined) { + group.diff = diff; + } + merged.mismatchLogging = group; + } + return merged as ShadowConfig; } function assertShadowConfig(config: ShadowConfig | undefined): void { if (config !== undefined && (config === null || typeof config !== "object" || Array.isArray(config))) { throw new TypeError("DialCache shadow config must be an object"); } + const mismatchLogging: unknown = config?.mismatchLogging; + if ( + mismatchLogging !== undefined + && (mismatchLogging === null || typeof mismatchLogging !== "object" || Array.isArray(mismatchLogging)) + ) { + throw new TypeError("DialCache shadow mismatchLogging config must be an object"); + } } + diff --git a/typescript/src/internal/shadow-log-json.ts b/typescript/src/internal/shadow-log-json.ts index a43b2dbd..1c5bdf2b 100644 --- a/typescript/src/internal/shadow-log-json.ts +++ b/typescript/src/internal/shadow-log-json.ts @@ -1,40 +1,428 @@ +import type { ShadowMismatchDiffLevel } from "../config.js"; +import { fulfills, isPromiseLike } from "./thenable.js"; + export const SHADOW_LOG_KEY_MAX_BYTES = 2 * 1024; export const SHADOW_LOG_VALUE_MAX_BYTES = 8 * 1024; +export const SHADOW_LOG_DIFF_MAX_BYTES = 8 * 1024; +export const SHADOW_MISMATCH_MAX_ENTRIES = 64; +export const SHADOW_MISMATCH_PATH_MAX_BYTES = 512; +export const SHADOW_MISMATCH_SIDE_MAX_BYTES = 1024; +/** Reference tokens in the deepest pointer the built-in walk reports. */ +export const SHADOW_MISMATCH_MAX_DEPTH = 32; export const SHADOW_LOG_TRUNCATION_MARKER = "...[truncated]"; const UTF8_ENCODER = new TextEncoder(); const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true }); const TRUNCATION_MARKER_BYTES = UTF8_ENCODER.encode(SHADOW_LOG_TRUNCATION_MARKER); -export interface ShadowMismatchLogDetails { - readonly cacheKey: string; - readonly cachedValueJson: string | null; - readonly sourceValueJson: string | null; +export type JsonValue = null | boolean | number | string | readonly JsonValue[] | JsonObject; +export interface JsonObject { + readonly [member: string]: JsonValue; +} + +/** + * One built-in difference: an RFC 6901 pointer into the compared value and + * the sides the walk holds there. A side is absent when the member exists on + * the other side only. + */ +export interface ShadowMismatchJsonEntry { + readonly path: string; + readonly cached?: JsonValue; + readonly source?: JsonValue; +} + +/** The capped built-in difference list and its uncut length. */ +export interface ShadowMismatchJsonDiff { + readonly entries: readonly ShadowMismatchJsonEntry[]; + readonly count: number; +} + +/** The effective content leaves of one confirmed-mismatch warning. */ +export interface ShadowMismatchLogPlan { + readonly value: boolean; + readonly diff: ShadowMismatchDiffLevel; +} + +/** A synchronous replacement for the built-in difference computation. */ +export type ShadowMismatchEntriesHook = (cachedValue: never, sourceValue: never) => unknown; + +/** Bounded content fields; each requested field is present, as null when it failed closed. */ +export interface ShadowMismatchLogFields { + readonly mismatchPaths?: readonly string[] | null; + readonly mismatchPathCount?: number | null; + readonly diffJson?: string | null; + readonly cachedValueJson?: string | null; + readonly sourceValueJson?: string | null; +} + +/** + * Why warning construction failed: `hook_error` when the hook threw or + * returned something other than an array, which ranks above `render_error` + * for JSON rendering. `null` when every requested field rendered. + */ +export type ShadowMismatchLogFailure = "hook_error" | "render_error"; + +export interface ShadowMismatchLogContent { + readonly fields: ShadowMismatchLogFields; + readonly failure: ShadowMismatchLogFailure | null; } export function previewShadowLogKey(value: string): string { return clampUtf8(value, SHADOW_LOG_KEY_MAX_BYTES); } -export function previewShadowLogJson(value: unknown): string | null { +/** + * Builds the content fields the plan requests for one confirmed mismatch. + * Each side is rendered to native JSON at most once, and every built-in field + * derives from that rendering, so a stateful `toJSON` cannot put into one + * field what another omits. Every failure fails its fields closed to `null`; + * a failed hook never falls back to the built-in computation. An absent side, + * the top-level `undefined` a negative-cache entry stores, is not a failure: + * its value field is `null`, and the built-in reports the other side whole at + * the root. + */ +export function renderShadowMismatchLog( + plan: ShadowMismatchLogPlan, + cachedValue: unknown, + sourceValue: unknown, + hook?: ShadowMismatchEntriesHook, +): ShadowMismatchLogContent { + const renderSides = plan.value || (plan.diff !== "off" && hook === undefined); + const cachedJson = renderSides ? renderJson(cachedValue) : null; + const sourceJson = renderSides ? renderJson(sourceValue) : null; + let diffFailed = false; + let renderFailed = renderSides + && ((cachedJson === null && cachedValue !== undefined) || (sourceJson === null && sourceValue !== undefined)); + + let diffFields: ShadowMismatchLogFields = {}; + if (plan.diff !== "off") { + const withValues = plan.diff === "values"; + let diff: RawMismatchDiff | null = null; + if (hook !== undefined) { + diff = runShadowMismatchHook(hook, cachedValue, sourceValue); + diffFailed = diff === null; + } else if (!renderFailed) { + diff = walkRenderedJson(cachedJson, sourceJson); + } + + if (diff === null) { + diffFields = { + mismatchPaths: null, + mismatchPathCount: null, + ...(withValues ? { diffJson: null } : {}), + }; + } else { + let diffJson: string | null = null; + if (withValues) { + diffJson = renderDiffJson(diff.entries); + renderFailed ||= diffJson === null; + } + diffFields = { + // "paths" never renders a side. + mismatchPaths: diff.entries.map((entry) => clampPointer(entry.path)), + mismatchPathCount: diff.count, + ...(withValues ? { diffJson } : {}), + }; + } + } + + return { + fields: { + ...diffFields, + ...(plan.value + ? { + cachedValueJson: cachedJson === null ? null : clampUtf8(cachedJson, SHADOW_LOG_VALUE_MAX_BYTES), + sourceValueJson: sourceJson === null ? null : clampUtf8(sourceJson, SHADOW_LOG_VALUE_MAX_BYTES), + } + : {}), + }, + failure: diffFailed ? "hook_error" : renderFailed ? "render_error" : null, + }; +} + +/** Every field the plan requests, failed closed to `null`, in rendering order. */ +export function failedShadowMismatchLogFields(plan: ShadowMismatchLogPlan): ShadowMismatchLogFields { + return { + ...(plan.diff === "off" ? {} : { mismatchPaths: null, mismatchPathCount: null }), + ...(plan.diff === "values" ? { diffJson: null } : {}), + ...(plan.value ? { cachedValueJson: null, sourceValueJson: null } : {}), + }; +} + +/** + * The built-in difference computation over two JSON values. Objects recurse + * over the union of their member names in UTF-16 code-unit order, and a member + * present on one side only is reported at its pointer with that side. Arrays + * of equal length recurse by index; arrays of different length are reported + * once at their own pointer. Any other differing pair, roots of different kinds + * included, is reported at its pointer with both sides, and a differing pair of + * containers at `SHADOW_MISMATCH_MAX_DEPTH` reference tokens is reported whole + * without descending. Entries keep walk order and are cut at + * `SHADOW_MISMATCH_MAX_ENTRIES`; `count` is the uncut total. Each pointer is + * clamped to `SHADOW_MISMATCH_PATH_MAX_BYTES`, and a side whose JSON exceeds + * `SHADOW_MISMATCH_SIDE_MAX_BYTES` becomes the clamped JSON text as a string. + */ +export function diffShadowMismatchJson(cached: JsonValue, source: JsonValue): ShadowMismatchJsonDiff { + const diff = walkJson(cached, source); + // JSON values always render, so every entry bounds. + return { count: diff.count, entries: diff.entries.map((entry) => boundEntry(entry)!) }; +} + +/** Entries before the pointer and side clamps, from the walk or the hook. */ +interface RawMismatchDiff { + readonly entries: readonly RawMismatchEntry[]; + readonly count: number; +} + +interface RawMismatchEntry { + readonly path: string; + readonly cached?: unknown; + readonly source?: unknown; +} + +/** `null` is an absent side; a side that failed to render never reaches here. */ +function walkRenderedJson(cachedJson: string | null, sourceJson: string | null): RawMismatchDiff { + if (cachedJson === sourceJson) { + return { entries: [], count: 0 }; + } + // Each string came from successful native JSON rendering, so this is the + // single boundary from arbitrary values to the closed JSON domain. + if (cachedJson === null) { + return { entries: [{ path: "", source: JSON.parse(sourceJson!) as JsonValue }], count: 1 }; + } + if (sourceJson === null) { + return { entries: [{ path: "", cached: JSON.parse(cachedJson) as JsonValue }], count: 1 }; + } + return walkJson(JSON.parse(cachedJson) as JsonValue, JSON.parse(sourceJson) as JsonValue); +} + +function walkJson(cached: JsonValue, source: JsonValue): ShadowMismatchJsonDiff { + const entries: ShadowMismatchJsonEntry[] = []; + // Reference tokens from the root to the current pair; a pointer is built + // only for an entry the cut keeps. + const tokens: (number | string)[] = []; + let count = 0; + const report = (sides: { readonly cached?: JsonValue; readonly source?: JsonValue }): void => { + count += 1; + if (entries.length < SHADOW_MISMATCH_MAX_ENTRIES) { + entries.push({ path: keptPointer(tokens), ...sides }); + } + }; + const walk = (left: JsonValue, right: JsonValue): void => { + const kind = jsonKind(left); + if (kind !== jsonKind(right) || kind === "primitive") { + if (left !== right) { + report({ cached: left, source: right }); + } + return; + } + if (tokens.length >= SHADOW_MISMATCH_MAX_DEPTH) { + if (!jsonValuesEqual(left, right)) { + report({ cached: left, source: right }); + } + return; + } + if (kind === "array") { + const leftItems = left as readonly JsonValue[]; + const rightItems = right as readonly JsonValue[]; + if (leftItems.length !== rightItems.length) { + report({ cached: left, source: right }); + return; + } + for (let index = 0; index < leftItems.length; index += 1) { + tokens.push(index); + walk(leftItems[index]!, rightItems[index]!); + tokens.pop(); + } + return; + } + const leftMembers = left as JsonObject; + const rightMembers = right as JsonObject; + const names = Object.keys(leftMembers); + for (const name of Object.keys(rightMembers)) { + if (!Object.hasOwn(leftMembers, name)) { + names.push(name); + } + } + // The default sort order is UTF-16 code units. + names.sort(); + for (const name of names) { + tokens.push(name); + if (!Object.hasOwn(rightMembers, name)) { + report({ cached: leftMembers[name]! }); + } else if (!Object.hasOwn(leftMembers, name)) { + report({ source: rightMembers[name]! }); + } else { + walk(leftMembers[name]!, rightMembers[name]!); + } + tokens.pop(); + } + }; + walk(cached, source); + return { entries, count }; +} + +// Structural equality for a depth-capped pair, iterative so that a subtree as +// deep as native JSON rendering allows cannot exhaust the stack. +function jsonValuesEqual(cached: JsonValue, source: JsonValue): boolean { + const pending: [JsonValue, JsonValue][] = [[cached, source]]; + while (pending.length > 0) { + const [left, right] = pending.pop()!; + const kind = jsonKind(left); + if (kind !== jsonKind(right)) { + return false; + } + if (kind === "primitive") { + if (left !== right) { + return false; + } + } else if (kind === "array") { + const leftItems = left as readonly JsonValue[]; + const rightItems = right as readonly JsonValue[]; + if (leftItems.length !== rightItems.length) { + return false; + } + for (let index = 0; index < leftItems.length; index += 1) { + pending.push([leftItems[index]!, rightItems[index]!]); + } + } else { + const leftMembers = left as JsonObject; + const rightMembers = right as JsonObject; + const names = Object.keys(leftMembers); + if (names.length !== Object.keys(rightMembers).length) { + return false; + } + for (const name of names) { + if (!Object.hasOwn(rightMembers, name)) { + return false; + } + pending.push([leftMembers[name]!, rightMembers[name]!]); + } + } + } + return true; +} + +function jsonKind(value: JsonValue): "array" | "object" | "primitive" { + if (Array.isArray(value)) { + return "array"; + } + return value !== null && typeof value === "object" ? "object" : "primitive"; +} + +/** + * Joins reference tokens into a pointer, building no more of it than the + * pointer clamp can keep. Each UTF-16 code unit is at least one UTF-8 byte, so + * a pointer longer than `SHADOW_MISMATCH_PATH_MAX_BYTES` code units is clamped + * to leading bytes that no later token changes: it stops growing there, and a + * member name is cut to that many code units before escaping. The clamped + * pointer is unchanged, and each kept entry materializes at most three times + * the cap however deep it is or however long the member names above it are. + */ +function keptPointer(tokens: readonly (number | string)[]): string { + let path = ""; + for (const token of tokens) { + if (path.length > SHADOW_MISMATCH_PATH_MAX_BYTES) { + break; + } + path += typeof token === "number" + ? `/${token}` + : `/${escapePointerToken(token.slice(0, SHADOW_MISMATCH_PATH_MAX_BYTES))}`; + } + return path; +} + +function escapePointerToken(name: string): string { + return name.replaceAll("~", "~0").replaceAll("/", "~1"); +} + +/** + * Runs the use case's hook under the comparator's synchronous discipline. A + * throw, a promise-like or a non-array returns `null`. Only the kept entries + * are read, inside the catch, and a path that is not a string is logged as + * its string form: a hook that returns wrong entries gets wrong paths. + */ +function runShadowMismatchHook( + hook: ShadowMismatchEntriesHook, + cachedValue: unknown, + sourceValue: unknown, +): RawMismatchDiff | null { try { - const json = JSON.stringify(value); - return json === undefined ? null : clampUtf8(json, SHADOW_LOG_VALUE_MAX_BYTES); + const result: unknown = (hook as (cached: unknown, source: unknown) => unknown)(cachedValue, sourceValue); + if (isPromiseLike(result)) { + // The contract is synchronous. Consume an accidental rejecting + // thenable without awaiting it, and fail closed. + void fulfills(result); + return null; + } + if (!Array.isArray(result)) { + return null; + } + const entries: RawMismatchEntry[] = []; + for (const member of result.slice(0, SHADOW_MISMATCH_MAX_ENTRIES) as readonly RawMismatchEntry[]) { + const { path, cached, source } = member; + entries.push({ + path: String(path), + ...(cached === undefined ? {} : { cached }), + ...(source === undefined ? {} : { source }), + }); + } + return { entries, count: result.length }; } catch { return null; } } -export function shadowMismatchLogDetails( - cacheKey: string, - cachedValue: unknown, - sourceValue: unknown, -): ShadowMismatchLogDetails { - return { - cacheKey: previewShadowLogKey(cacheKey), - cachedValueJson: previewShadowLogJson(cachedValue), - sourceValueJson: previewShadowLogJson(sourceValue), - }; +/** + * Renders entries with their bounded sides; a side with no JSON rendering + * makes the whole field `null`, and later sides are not rendered. + */ +function renderDiffJson(entries: readonly RawMismatchEntry[]): string | null { + const bounded: ShadowMismatchJsonEntry[] = []; + for (const entry of entries) { + const next = boundEntry(entry); + if (next === null) { + return null; + } + bounded.push(next); + } + return clampUtf8(JSON.stringify(bounded), SHADOW_LOG_DIFF_MAX_BYTES); +} + +/** + * The one clamp rule for built-in and hook entries: the pointer is clamped, + * and each present side is rendered once and kept as JSON, or as its clamped + * text when the rendering exceeds the side cap. `null` when a side has no + * JSON rendering. + */ +function boundEntry(entry: RawMismatchEntry): ShadowMismatchJsonEntry | null { + const bounded: { path: string; cached?: JsonValue; source?: JsonValue } = { path: clampPointer(entry.path) }; + for (const side of ["cached", "source"] as const) { + if (!Object.hasOwn(entry, side)) { + continue; + } + const json = renderJson(entry[side]); + if (json === null) { + return null; + } + const clamped = clampUtf8(json, SHADOW_MISMATCH_SIDE_MAX_BYTES); + // Parsing the one rendering keeps a stateful `toJSON` from rendering twice. + bounded[side] = clamped === json ? JSON.parse(json) as JsonValue : clamped; + } + return bounded; +} + +function clampPointer(path: string): string { + return clampUtf8(path, SHADOW_MISMATCH_PATH_MAX_BYTES); +} + +function renderJson(value: unknown): string | null { + try { + const json = JSON.stringify(value); + return json === undefined ? null : json; + } catch { + return null; + } } function clampUtf8(value: string, maxBytes: number): string { diff --git a/typescript/src/internal/thenable.ts b/typescript/src/internal/thenable.ts new file mode 100644 index 00000000..ce36852e --- /dev/null +++ b/typescript/src/internal/thenable.ts @@ -0,0 +1,21 @@ +// Application callbacks under a synchronous contract (comparators, the stale +// recovery predicate, the mismatch hook) and observers (loggers, metrics +// adapters) can still return a promise-like. These helpers observe its +// settlement so a rejection never goes unhandled. + +/** Whether a value has a callable `then`, which `await` would adopt. */ +export function isPromiseLike(value: unknown): value is PromiseLike { + return value !== null + && (typeof value === "object" || typeof value === "function") + && typeof (value as { readonly then?: unknown }).then === "function"; +} + +/** Resolves to whether `value` fulfilled. Never rejects, so a rejection is never left unhandled. */ +export async function fulfills(value: unknown): Promise { + try { + await value; + return true; + } catch { + return false; + } +} diff --git a/typescript/src/metrics.ts b/typescript/src/metrics.ts index daea9920..b23ab146 100644 --- a/typescript/src/metrics.ts +++ b/typescript/src/metrics.ts @@ -25,6 +25,20 @@ export type ShadowValidationOutcome = | "confirmation_error" | "timeout" | "dropped"; +/** + * Bounded outcomes of one confirmed-mismatch warning attempt. `hook_error`: + * the use case's `shadowMismatchDiff` hook threw or returned something other + * than an array. + * `render_error`: a value had no JSON rendering. + * `logger_error`: the logger threw, or the promise it returned rejected. When + * several fail, `hook_error` ranks above `render_error`, which ranks above + * `logger_error`. + */ +export type ShadowMismatchLoggingOutcome = + | "logged" + | "hook_error" + | "render_error" + | "logger_error"; /** Bounded terminal outcomes for an attempted stale-on-error Redis recovery. */ export type StaleRecoveryOutcome = | "served" @@ -133,6 +147,13 @@ export interface ShadowValidationMetricLabels { readonly outcome: ShadowValidationOutcome; } +export interface ShadowMismatchLoggingMetricLabels { + readonly cacheNamespace: string; + readonly useCase: string; + readonly keyType: string; + readonly outcome: ShadowMismatchLoggingOutcome; +} + export interface StaleRecoveryMetricLabels { readonly cacheNamespace: string; readonly useCase: string; @@ -161,6 +182,12 @@ export interface DialCacheMetricsAdapter { * Optional so existing custom adapters keep compiling without changes. */ observeShadowValueAge?(labels: ShadowValidationMetricLabels, seconds: number): void; + /** + * One event per confirmed-mismatch warning attempt, emitted only when + * mismatch logging is on. Optional so existing custom adapters keep + * compiling without changes. + */ + shadowMismatchLogging?(labels: ShadowMismatchLoggingMetricLabels): void; /** * Positive offset in seconds when a decoded Redis frame is dated after the * observing process's epoch clock. Serving and initial shadow reads fail diff --git a/typescript/src/prometheus.ts b/typescript/src/prometheus.ts index 25e40468..af8dd0c8 100644 --- a/typescript/src/prometheus.ts +++ b/typescript/src/prometheus.ts @@ -11,6 +11,7 @@ import type { InvalidationMetricLabels, MissMetricLabels, SerializationMetricLabels, + ShadowMismatchLoggingMetricLabels, ShadowValidationMetricLabels, StaleRecoveryMetricLabels, } from "./metrics.js"; @@ -30,7 +31,10 @@ type SerializationLabels = CounterLabels | "operation"; type InvalidationLabels = "cache_namespace" | "key_type" | "layer"; type CoalescedLabels = "cache_namespace" | "use_case" | "key_type" | "scope"; type OutcomeLabels = "cache_namespace" | "use_case" | "key_type" | "outcome"; -type OutcomeMetricLabels = ShadowValidationMetricLabels | StaleRecoveryMetricLabels; +type OutcomeMetricLabels = + | ShadowValidationMetricLabels + | ShadowMismatchLoggingMetricLabels + | StaleRecoveryMetricLabels; type CompressionLabels = CounterLabels | "outcome"; interface BaseCollectorConfig { @@ -93,6 +97,7 @@ export class PrometheusDialCacheMetrics implements DialCacheMetricsAdapter { private readonly coalescedCounter: Counter; private readonly shadowValidationCounter: Counter; private readonly shadowValueAgeHistogram: Histogram; + private readonly shadowMismatchLoggingCounter: Counter; private readonly futureTimestampOffsetHistogram: Histogram; private readonly staleRecoveryCounter: Counter; private readonly staleRecoveryValueAgeHistogram: Histogram; @@ -119,6 +124,7 @@ export class PrometheusDialCacheMetrics implements DialCacheMetricsAdapter { this.coalescedCounter = counter(registry, collectors.coalescedCounter); this.shadowValidationCounter = counter(registry, collectors.shadowValidationCounter); this.shadowValueAgeHistogram = histogram(registry, collectors.shadowValueAgeHistogram); + this.shadowMismatchLoggingCounter = counter(registry, collectors.shadowMismatchLoggingCounter); this.futureTimestampOffsetHistogram = histogram(registry, collectors.futureTimestampOffsetHistogram); this.staleRecoveryCounter = counter(registry, collectors.staleRecoveryCounter); this.staleRecoveryValueAgeHistogram = histogram(registry, collectors.staleRecoveryValueAgeHistogram); @@ -177,6 +183,10 @@ export class PrometheusDialCacheMetrics implements DialCacheMetricsAdapter { this.shadowValueAgeHistogram.observe(outcomeLabels(labels), seconds); } + shadowMismatchLogging(labels: ShadowMismatchLoggingMetricLabels): void { + this.shadowMismatchLoggingCounter.inc(outcomeLabels(labels)); + } + observeFutureTimestampOffset(labels: CacheMetricLabels, seconds: number): void { if (!Number.isFinite(seconds) || seconds <= 0) { return; @@ -298,6 +308,12 @@ function collectorConfigs(prefix: string) { labelNames: ["cache_namespace", "use_case", "key_type", "outcome"], buckets: VALUE_AGE_BUCKETS, }, + shadowMismatchLoggingCounter: { + type: "counter", + name: `${prefix}dialcache_shadow_mismatch_logging_counter`, + help: "DialCache confirmed shadow-mismatch warning outcomes.", + labelNames: ["cache_namespace", "use_case", "key_type", "outcome"], + }, futureTimestampOffsetHistogram: { type: "histogram", name: `${prefix}dialcache_future_timestamp_offset_histogram`, diff --git a/typescript/test/datadog.test.ts b/typescript/test/datadog.test.ts index 626cbc23..4546d13a 100644 --- a/typescript/test/datadog.test.ts +++ b/typescript/test/datadog.test.ts @@ -11,6 +11,7 @@ import { type DialCacheRedisClient, type MetricErrorKind, type MetricLayer, + type ShadowMismatchLoggingOutcome, type ShadowValidationOutcome, type StaleRecoveryOutcome, } from "../src/index.js"; @@ -130,6 +131,13 @@ const SHADOW_VALIDATION_OUTCOMES: Readonly dropped: true, }; const shadowValidationOutcomes = Object.keys(SHADOW_VALIDATION_OUTCOMES) as ShadowValidationOutcome[]; +const SHADOW_MISMATCH_LOGGING_OUTCOMES: Readonly> = { + logged: true, + hook_error: true, + render_error: true, + logger_error: true, +}; +const shadowMismatchLoggingOutcomes = Object.keys(SHADOW_MISMATCH_LOGGING_OUTCOMES) as ShadowMismatchLoggingOutcome[]; const STALE_RECOVERY_OUTCOMES: Readonly> = { served: true, miss: true, @@ -200,6 +208,12 @@ describe("Datadog metrics adapter", () => { }, 42.5, ); + metrics.shadowMismatchLogging({ + cacheNamespace: cacheLabels.cacheNamespace, + useCase: "LoadUser", + keyType: "user_id", + outcome: "logged", + }); metrics.observeFutureTimestampOffset(cacheLabels, 0.007); metrics.compression({ ...cacheLabels, outcome: "compressed" }); metrics.observeGet(cacheLabels, 0.125); @@ -267,6 +281,12 @@ describe("Datadog metrics adapter", () => { value: 42.5, tags: { cache_namespace: "users", use_case: "LoadUser", key_type: "user_id", outcome: "mismatch" }, }, + { + method: "increment", + name: "dialcache.shadow_mismatch_logging.count", + value: 1, + tags: { cache_namespace: "users", use_case: "LoadUser", key_type: "user_id", outcome: "logged" }, + }, { method: "distribution", name: "dialcache.future_timestamp_offset", @@ -388,6 +408,14 @@ describe("Datadog metrics adapter", () => { outcome, }); } + for (const outcome of shadowMismatchLoggingOutcomes) { + metrics.shadowMismatchLogging({ + cacheNamespace: cacheLabels.cacheNamespace, + useCase: cacheLabels.useCase, + keyType: cacheLabels.keyType, + outcome, + }); + } for (const outcome of staleRecoveryOutcomes) { metrics.staleRecovery({ cacheNamespace: cacheLabels.cacheNamespace, @@ -438,6 +466,18 @@ describe("Datadog metrics adapter", () => { outcome, })), ); + expect( + client.calls + .filter(({ name }) => name === "dialcache.shadow_mismatch_logging.count") + .map(({ tags }) => tags), + ).toEqual( + shadowMismatchLoggingOutcomes.map((outcome) => ({ + cache_namespace: cacheLabels.cacheNamespace, + use_case: cacheLabels.useCase, + key_type: cacheLabels.keyType, + outcome, + })), + ); expect( client.calls .filter(({ name }) => name === "dialcache.stale_recovery.count") @@ -537,8 +577,15 @@ describe("Datadog metrics adapter", () => { keyType: "user_id", outcome: "match", }) as unknown as Promise; + const loggingReturned = metrics.shadowMismatchLogging({ + cacheNamespace: "users", + useCase: "AsyncShadowMetric", + keyType: "user_id", + outcome: "logged", + }) as unknown as Promise; await expect(returned).rejects.toBe(metricError); + await expect(loggingReturned).rejects.toBe(metricError); }); it("never emits cache ids, arguments, Redis keys, or raw errors as Datadog tags", async () => { @@ -670,24 +717,23 @@ describe("Datadog metrics adapter", () => { it("enforces Datadog's 200-character final metric-name limit", () => { const client = new RecordingDogStatsDClient(); - const longestValidNamespace = "a".repeat(175); - const tooLongNamespace = "a".repeat(176); + // shadow_mismatch_logging.count is the longest suffix: 170 + "." + 29 = 200. + const longestValidNamespace = "a".repeat(170); + const tooLongNamespace = "a".repeat(171); const metrics = new DatadogDialCacheMetrics({ client, namespace: longestValidNamespace, observationMetricType: "distribution", }); - metrics.observeStaleRecoveryValueAge( - { - cacheNamespace: cacheLabels.cacheNamespace, - useCase: cacheLabels.useCase, - keyType: cacheLabels.keyType, - outcome: "served", - }, - 1, - ); + metrics.shadowMismatchLogging({ + cacheNamespace: cacheLabels.cacheNamespace, + useCase: cacheLabels.useCase, + keyType: cacheLabels.keyType, + outcome: "logged", + }); + expect(client.calls[0]?.name).toBe(`${longestValidNamespace}.shadow_mismatch_logging.count`); expect(client.calls[0]?.name).toHaveLength(200); expect( () => diff --git a/typescript/test/dialcache-config-ramp.test.ts b/typescript/test/dialcache-config-ramp.test.ts index a885b060..01140ea4 100644 --- a/typescript/test/dialcache-config-ramp.test.ts +++ b/typescript/test/dialcache-config-ramp.test.ts @@ -7,6 +7,7 @@ import { DialCacheKeyConfig, type LayerConfig, type Serializer, + type ShadowMismatchDiffLevel, } from "../src/index.js"; import { deterministicRampSample, deterministicShadowRampSample } from "../src/internal/ramp.js"; import { FakeRedis } from "./fake-redis.js"; @@ -81,6 +82,23 @@ describe("DialCache runtime config and ramp controls", () => { }); }); + it("clones the supplied mismatchLogging leaves", () => { + const suppliedLeaves: { value: boolean; diff: ShadowMismatchDiffLevel } = { value: true, diff: "paths" }; + const config = new DialCacheKeyConfig({ shadow: { mismatchLogging: suppliedLeaves } }); + + suppliedLeaves.value = false; + suppliedLeaves.diff = "off"; + + expect(config.shadow?.mismatchLogging).not.toBe(suppliedLeaves); + expect(config.shadow).toEqual({ mismatchLogging: { value: true, diff: "paths" } }); + }); + + it("leaves mismatchLogging leaf values unvalidated until static capture or runtime resolution", () => { + expect(new DialCacheKeyConfig({ + shadow: { mismatchLogging: { value: "yes", diff: "full" } }, + } as never).shadow).toStrictEqual({ mismatchLogging: { value: "yes", diff: "full" } }); + }); + it("captures an immutable default policy snapshot when the use case is registered", async () => { const suppliedDefault = new DialCacheKeyConfig({ ttlSec: { [CacheLayer.LOCAL]: 60, [CacheLayer.REMOTE]: 60 }, @@ -139,6 +157,36 @@ describe("DialCache runtime config and ramp controls", () => { expect(Object.isFrozen(observedDefaults[0]?.shadow)).toBe(true); }); + it("captures an immutable mismatchLogging snapshot when the use case is registered", async () => { + const suppliedDefault = new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.LOCAL]: 60 }, + shadow: { mismatchLogging: { value: true, diff: "paths" } }, + }); + const observedDefaults: Array = []; + const dialcache = new DialCache({ + cacheConfigProvider: (key) => { + observedDefaults.push(key.defaultConfig); + return null; + }, + }); + const getUser = dialcache.cached(async (userId: string) => ({ userId }), { + keyType: "user_id", + useCase: "ImmutableRegisteredMismatchLogging", + cacheKey: (userId) => userId, + defaultConfig: suppliedDefault, + }); + + const suppliedLeaves = suppliedDefault.shadow?.mismatchLogging as { value?: boolean; diff?: ShadowMismatchDiffLevel }; + suppliedLeaves.value = false; + suppliedLeaves.diff = "off"; + await dialcache.enable(async () => await getUser("123")); + + const snapshotLeaves = observedDefaults[0]?.shadow?.mismatchLogging; + expect(snapshotLeaves).not.toBe(suppliedLeaves); + expect(snapshotLeaves).toEqual({ value: true, diff: "paths" }); + expect(Object.isFrozen(snapshotLeaves)).toBe(true); + }); + it("enables request-local caching without TTL or ramp policy", async () => { const dialcache = new DialCache(); let calls = 0; @@ -348,6 +396,21 @@ describe("DialCache runtime config and ramp controls", () => { () => new DialCacheKeyConfig({ shadowRamp: 100 } as never), 'DialCacheKeyConfig.shadowRamp was replaced by "shadow.ramp"', ], + [ + "a null mismatchLogging config", + () => new DialCacheKeyConfig({ shadow: { mismatchLogging: null as never } }), + "DialCache shadow mismatchLogging config must be an object", + ], + [ + "an array mismatchLogging config", + () => new DialCacheKeyConfig({ shadow: { mismatchLogging: [] as never } }), + "DialCache shadow mismatchLogging config must be an object", + ], + [ + "a primitive mismatchLogging config", + () => new DialCacheKeyConfig({ shadow: { mismatchLogging: "paths" as never } }), + "DialCache shadow mismatchLogging config must be an object", + ], ])("rejects $0 in the public config constructor", (_name, construct, message) => { expect(construct).toThrow(message); }); @@ -483,6 +546,48 @@ describe("DialCache runtime config and ramp controls", () => { TypeError, "must be a boolean", ], + [ + "wrong-type shadow mismatchLogging value", + new DialCacheKeyConfig({ shadow: { mismatchLogging: { value: "true" as unknown as boolean } } }), + TypeError, + "shadow.mismatchLogging.value must be a boolean", + ], + [ + "a null shadow mismatchLogging value", + new DialCacheKeyConfig({ shadow: { mismatchLogging: { value: null as unknown as boolean } } }), + TypeError, + "shadow.mismatchLogging.value must be a boolean", + ], + [ + "an unknown shadow mismatchLogging diff level", + new DialCacheKeyConfig({ shadow: { mismatchLogging: { diff: "full" as unknown as ShadowMismatchDiffLevel } } }), + TypeError, + 'shadow.mismatchLogging.diff must be "off", "paths" or "values"', + ], + [ + "a shadow mismatchLogging diff naming an Object.prototype member", + new DialCacheKeyConfig({ shadow: { mismatchLogging: { diff: "toString" as unknown as ShadowMismatchDiffLevel } } }), + TypeError, + 'shadow.mismatchLogging.diff must be "off", "paths" or "values"', + ], + [ + "wrong-type shadow mismatchLogging diff", + new DialCacheKeyConfig({ shadow: { mismatchLogging: { diff: true as unknown as ShadowMismatchDiffLevel } } }), + TypeError, + 'shadow.mismatchLogging.diff must be "off", "paths" or "values"', + ], + [ + "a null shadow mismatchLogging group", + { ttlSec: {}, ramp: {}, shadow: { mismatchLogging: null } } as unknown as DialCacheKeyConfig, + TypeError, + "shadow mismatchLogging config must be an object", + ], + [ + "an array shadow mismatchLogging group", + { ttlSec: {}, ramp: {}, shadow: { mismatchLogging: [] } } as unknown as DialCacheKeyConfig, + TypeError, + "shadow mismatchLogging config must be an object", + ], [ "wrong-type stale-on-error max age", new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: "3600" as unknown as number }), @@ -578,6 +683,8 @@ describe("DialCache runtime config and ramp controls", () => { ["the removed shadowRamp field", { ttlSec: {}, ramp: {}, shadowRamp: 100 }], ["a null requestLocal value", { ttlSec: {}, ramp: {}, requestLocal: null }], ["a null coalesce value", { ttlSec: {}, ramp: {}, coalesce: null }], + ["a null mismatchLogging group", { ttlSec: {}, ramp: {}, shadow: { mismatchLogging: null } }], + ["an array mismatchLogging group", { ttlSec: {}, ramp: {}, shadow: { mismatchLogging: [] } }], ] as const)("fails open instead of inheriting defaults when the provider returns %s", async (_name, runtimeConfig) => { const cacheConfigProvider = vi.fn(async () => runtimeConfig as unknown as DialCacheKeyConfig); const dialcache = new DialCache({ cacheConfigProvider }); @@ -603,10 +710,13 @@ describe("DialCache runtime config and ramp controls", () => { staleOnErrorMaxAgeSec: 0, shadow: { ramp: 0, - logMismatches: false, + mismatchLogging: { value: false, diff: "off" }, }, ramp: { [CacheLayer.LOCAL]: 0, [CacheLayer.REMOTE]: 0 }, })); + // Both leaves are written, so neither can inherit a default leaf through + // the leaf-wise merge, and the shorthand is not needed. + expect(DialCacheKeyConfig.disabled().shadow).not.toHaveProperty("logMismatches"); // disabled() must not pin coalescing off: a disabled() baseline ramped up // at runtime should coalesce again unless explicitly opted out. expect(DialCacheKeyConfig.disabled().coalesce).toBeUndefined(); diff --git a/typescript/test/dialcache-metrics.test.ts b/typescript/test/dialcache-metrics.test.ts index ce1029a5..a4c53684 100644 --- a/typescript/test/dialcache-metrics.test.ts +++ b/typescript/test/dialcache-metrics.test.ts @@ -16,6 +16,7 @@ import { type RedisReadResult, type SerializationMetricLabels, type Serializer, + type ShadowMismatchLoggingMetricLabels, type ShadowValidationMetricLabels, type StaleRecoveryMetricLabels, } from "../src/index.js"; @@ -58,6 +59,14 @@ class RecordingMetrics implements DialCacheMetricsAdapter { this.record("coalesced", labels); } + shadowValidation(labels: ShadowValidationMetricLabels): void { + this.record("shadowValidation", labels); + } + + shadowMismatchLogging(labels: ShadowMismatchLoggingMetricLabels): void { + this.record("shadowMismatchLogging", labels); + } + staleRecovery(labels: StaleRecoveryMetricLabels): void { this.record("staleRecovery", labels); } @@ -125,6 +134,7 @@ describe("DialCache observability metrics", () => { invalidation: vi.fn(() => thenable), coalesced: vi.fn(() => thenable), shadowValidation: vi.fn(() => thenable), + shadowMismatchLogging: vi.fn(() => thenable), observeFutureTimestampOffset: vi.fn(() => thenable), staleRecovery: vi.fn(() => thenable), observeStaleRecoveryValueAge: vi.fn(() => thenable), @@ -165,6 +175,12 @@ describe("DialCache observability metrics", () => { keyType: "user_id", outcome: "match", } satisfies ShadowValidationMetricLabels); + isolatedMetrics.shadowMismatchLogging?.({ + cacheNamespace: "urn", + useCase: "RejectingMetricsThenable", + keyType: "user_id", + outcome: "logged", + } satisfies ShadowMismatchLoggingMetricLabels); isolatedMetrics.observeFutureTimestampOffset?.(labels, 0.001); isolatedMetrics.staleRecovery?.({ cacheNamespace: "urn", @@ -188,7 +204,7 @@ describe("DialCache observability metrics", () => { expect(then).not.toHaveBeenCalled(); await tick(); - expect(then).toHaveBeenCalledTimes(14); + expect(then).toHaveBeenCalledTimes(15); }); it("includes the configured cache namespace on every metric path", async () => { @@ -702,6 +718,55 @@ describe("DialCache observability metrics", () => { ]); }); + it("fails open when the shadow mismatch logging metrics hook throws", async () => { + const metrics = new RecordingMetrics(); + const shadowMismatchLogging = vi.spyOn(metrics, "shadowMismatchLogging").mockImplementation(() => { + throw new Error("metrics unavailable"); + }); + const redis = new FakeRedis(); + const useCase = "ThrowingShadowMismatchLoggingMetricFailOpen"; + const cachedValue = { userId: "123", version: 1 }; + const key = new DialCacheKey({ keyType: "user_id", id: "123", useCase }); + redis.setRaw(`${key.urn}:dialcache-frame-v1`, encodeFrame(cachedValue), 60_000); + const logger = { debug: vi.fn(), warn: vi.fn(), error: vi.fn() }; + const dialcache = new DialCache({ metrics, redis: { client: redis, readTimeoutMs: 1_000 }, logger }); + const getUser = dialcache.cached(async () => ({ userId: "123", version: 2 }), { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + shadow: { ramp: 100, mismatchLogging: { diff: "paths" } }, + }), + }); + + // A throwing hook after the first warning must leave the next served hit and validation intact. + for (const attempt of [1, 2]) { + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(cachedValue); + await vi.waitFor(() => { + expect(shadowMismatchLogging).toHaveBeenCalledTimes(attempt); + }, { timeout: 1_000, interval: 1 }); + } + + const shadowLabels = { cacheNamespace: "urn", useCase, keyType: "user_id" }; + expect(events(metrics, "shadowValidation", { useCase })).toEqual([ + { name: "shadowValidation", labels: { ...shadowLabels, outcome: "mismatch" } }, + { name: "shadowValidation", labels: { ...shadowLabels, outcome: "mismatch" } }, + ]); + expect(shadowMismatchLogging.mock.calls).toEqual([ + [{ ...shadowLabels, outcome: "logged" }], + [{ ...shadowLabels, outcome: "logged" }], + ]); + expect(logger.warn).toHaveBeenCalledTimes(2); + expect(logger.warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + expect.objectContaining({ useCase, mismatchPaths: ["/version"], mismatchPathCount: 1 }), + ); + expect(logger.error).not.toHaveBeenCalled(); + expect(events(metrics, "error", { useCase })).toEqual([]); + }); + it("classifies disabled cache skips by reason", async () => { // Given one cache call is outside context and other enabled calls have disabled layer config. const metrics = new RecordingMetrics(); diff --git a/typescript/test/dialcache-observability-internals.test.ts b/typescript/test/dialcache-observability-internals.test.ts index 74ce725e..9b01c06e 100644 --- a/typescript/test/dialcache-observability-internals.test.ts +++ b/typescript/test/dialcache-observability-internals.test.ts @@ -1,14 +1,34 @@ import { describe, expect, it } from "vitest"; -import { CacheLayer, DialCacheKey, DialCacheKeyConfig } from "../src/index.js"; +import { CacheLayer, DialCacheKey, DialCacheKeyConfig, type ShadowConfig } from "../src/index.js"; import { + expandMismatchLogging, fetchKeyConfig, + resolveMismatchLogging, resolveRemoteLayerConfigResult, } from "../src/internal/runtime-config.js"; +/** A runtime shadow group as a provider may return it, its leaves unvalidated. */ +const rawShadow = (shadow: object): ShadowConfig => shadow as ShadowConfig; + const key = (defaultConfig: DialCacheKeyConfig | null = DialCacheKeyConfig.enabled(60)) => new DialCacheKey({ keyType: "user_id", id: "123", useCase: "ObservabilityInternals", defaultConfig }); +/** The shadow group fetchKeyConfig merges from one default and one runtime shadow group. */ +async function mergedShadow( + defaultShadow: ShadowConfig | undefined, + runtimeShadow: ShadowConfig, +): Promise { + const merged = await fetchKeyConfig( + async () => new DialCacheKeyConfig({ shadow: runtimeShadow }), + key(new DialCacheKeyConfig(defaultShadow === undefined ? {} : { shadow: defaultShadow })), + ); + if (merged?.shadow === undefined) { + throw new Error("expected a merged shadow group"); + } + return merged.shadow; +} + describe("DialCache observability internal compatibility paths", () => { it("merges every runtime policy leaf independently", async () => { const defaultConfig = new DialCacheKeyConfig({ @@ -36,12 +56,10 @@ describe("DialCache observability internal compatibility paths", () => { coalesce: true, ttlSec: { [CacheLayer.LOCAL]: 30, [CacheLayer.REMOTE]: 120 }, ramp: { [CacheLayer.LOCAL]: 25, [CacheLayer.REMOTE]: 75 }, - shadow: { - ramp: 80, - logMismatches: true, - }, + shadow: { ramp: 80, mismatchLogging: { value: true, diff: "values" } }, staleOnErrorMaxAgeSec: 3_600, }), + effective: { value: true, diff: "values" }, }, { runtime: new DialCacheKeyConfig({ @@ -57,12 +75,10 @@ describe("DialCache observability internal compatibility paths", () => { coalesce: false, ttlSec: { [CacheLayer.LOCAL]: 60, [CacheLayer.REMOTE]: 90 }, ramp: { [CacheLayer.LOCAL]: 10, [CacheLayer.REMOTE]: 50 }, - shadow: { - ramp: 20, - logMismatches: false, - }, + shadow: { ramp: 20, mismatchLogging: { value: false, diff: "off" } }, staleOnErrorMaxAgeSec: 0, }), + effective: { value: false, diff: "off" }, }, { runtime: new DialCacheKeyConfig({ shadow: {}, staleOnErrorMaxAgeSec: 7_200 }), @@ -71,17 +87,20 @@ describe("DialCache observability internal compatibility paths", () => { coalesce: false, ttlSec: { [CacheLayer.LOCAL]: 60, [CacheLayer.REMOTE]: 120 }, ramp: { [CacheLayer.LOCAL]: 25, [CacheLayer.REMOTE]: 50 }, - shadow: { - ramp: 20, - logMismatches: true, - }, + shadow: { ramp: 20, mismatchLogging: { value: true, diff: "values" } }, staleOnErrorMaxAgeSec: 7_200, }), + effective: { value: true, diff: "values" }, }, ]; - for (const { runtime, expected } of cases) { - await expect(fetchKeyConfig(async () => runtime, key(defaultConfig))).resolves.toEqual(expected); + for (const { runtime, expected, effective } of cases) { + const merged = await fetchKeyConfig(async () => runtime, key(defaultConfig)); + + expect(merged).toEqual(expected); + // Each layer's shorthand expands before the merge, so only leaves remain. + expect(merged?.shadow).not.toHaveProperty("logMismatches"); + expect(resolveMismatchLogging(merged!.shadow!)).toEqual({ ...effective, invalid: false }); } }); @@ -101,6 +120,279 @@ describe("DialCache observability internal compatibility paths", () => { expect(merged?.ramp[CacheLayer.LOCAL]).toBe(50); }); + it.each([ + [ + "a runtime true shorthand beside value: false", + undefined, + { logMismatches: true, mismatchLogging: { value: false } }, + { value: false, diff: "values" }, + ], + [ + "a default true shorthand beside value: false", + { logMismatches: true, mismatchLogging: { value: false } }, + {}, + { value: false, diff: "values" }, + ], + [ + "a runtime false shorthand beside diff: paths", + undefined, + { logMismatches: false, mismatchLogging: { diff: "paths" } }, + { value: false, diff: "paths" }, + ], + [ + "a default false shorthand beside diff: paths", + { logMismatches: false, mismatchLogging: { diff: "paths" } }, + {}, + { value: false, diff: "paths" }, + ], + [ + "a runtime false shorthand over a default diff: paths", + { mismatchLogging: { diff: "paths" } }, + { logMismatches: false }, + { value: false, diff: "off" }, + ], + [ + "a runtime true shorthand over a default value: false", + { mismatchLogging: { value: false } }, + { logMismatches: true }, + { value: true, diff: "values" }, + ], + ] as const)("resolves %s to the recorded effective leaves", async (_name, defaultShadow, runtimeShadow, effective) => { + const shadow = await mergedShadow(defaultShadow, runtimeShadow); + + expect(resolveMismatchLogging(shadow)).toEqual({ ...effective, invalid: false }); + }); + + it("inherits an omitted mismatchLogging leaf from the default", async () => { + const shadow = await mergedShadow( + { mismatchLogging: { value: true, diff: "paths" } }, + { mismatchLogging: { diff: "values" } }, + ); + + expect(shadow).toEqual({ mismatchLogging: { value: true, diff: "values" } }); + expect(resolveMismatchLogging(shadow)).toEqual({ value: true, diff: "values", invalid: false }); + }); + + it("lets a runtime mismatchLogging leaf override its default leaf", async () => { + await expect(mergedShadow( + { mismatchLogging: { value: true, diff: "values" } }, + { mismatchLogging: { value: false } }, + )).resolves.toEqual({ mismatchLogging: { value: false, diff: "values" } }); + // A default shorthand is a baseline like any other pair of default leaves. + const overShorthand = await mergedShadow({ logMismatches: true }, { mismatchLogging: { diff: "paths" } }); + expect(overShorthand).toEqual({ mismatchLogging: { value: true, diff: "paths" } }); + expect(resolveMismatchLogging(overShorthand)).toEqual({ value: true, diff: "paths", invalid: false }); + }); + + it("keeps a malformed runtime logMismatches for admission while its same-object leaves win", async () => { + const shadow = await mergedShadow( + { mismatchLogging: { value: false, diff: "values" } }, + { logMismatches: "yes" as unknown as boolean, mismatchLogging: { value: true } }, + ); + + // The malformed shorthand expands to both leaves off over the default, the + // leaf written beside it still wins, and the raw value stays so admission + // records its config_resolution. + expect(shadow).toEqual({ logMismatches: "yes", mismatchLogging: { value: true, diff: "off" } }); + expect(resolveMismatchLogging(shadow)).toEqual({ value: true, diff: "off", invalid: true }); + }); + + it.each([ + ["an omitted group", {}, { value: false, diff: "off", invalid: false }], + ["an empty group", { mismatchLogging: {} }, { value: false, diff: "off", invalid: false }], + [ + "undefined leaves", + { mismatchLogging: { value: undefined, diff: undefined } }, + { value: false, diff: "off", invalid: false }, + ], + ["valid leaves", { mismatchLogging: { value: true, diff: "paths" } }, { value: true, diff: "paths", invalid: false }], + [ + "a non-boolean value leaf", + { mismatchLogging: { value: "true", diff: "paths" } }, + { value: false, diff: "paths", invalid: true }, + ], + ["a null value leaf", { mismatchLogging: { value: null } }, { value: false, diff: "off", invalid: true }], + [ + "an unknown diff level", + { mismatchLogging: { value: true, diff: "full" } }, + { value: true, diff: "off", invalid: true }, + ], + ["a non-string diff leaf", { mismatchLogging: { diff: 1 } }, { value: false, diff: "off", invalid: true }], + [ + "a diff leaf naming an Object.prototype member", + { mismatchLogging: { diff: "toString" } }, + { value: false, diff: "off", invalid: true }, + ], + [ + "a diff leaf naming the constructor", + { mismatchLogging: { value: true, diff: "constructor" } }, + { value: true, diff: "off", invalid: true }, + ], + ["a malformed shorthand", { logMismatches: "yes" }, { value: false, diff: "off", invalid: true }], + ["a null shorthand", { logMismatches: null }, { value: false, diff: "off", invalid: true }], + [ + "a malformed shorthand beside both leaves", + { logMismatches: "yes", mismatchLogging: { value: true, diff: "values" } }, + { value: true, diff: "values", invalid: true }, + ], + ])("resolves %s to the effective mismatch-logging leaves", (_name, shadow, expected) => { + expect(resolveMismatchLogging(rawShadow(shadow))).toEqual(expected); + }); + + it("rejects the removed shadowRamp field in a provider result", async () => { + await expect(fetchKeyConfig( + async () => ({ ttlSec: {}, ramp: {}, shadowRamp: 100 }) as unknown as DialCacheKeyConfig, + key(), + )).rejects.toThrow('DialCacheKeyConfig.shadowRamp was replaced by "shadow.ramp"'); + }); + + it("resolves every merged combination of shorthand and leaves as each layer expanded first", async () => { + // The recorded rule, written out without the code under test: a written + // shorthand stands for both leaves (true: value on and diff "values"; false + // or malformed: both off), a leaf written beside it wins within its own + // layer, the runtime layer wins per leaf, an omitted leaf is off, an invalid + // leaf acts off, and an invalid input owes one config_resolution. Defaults + // stay valid, as registration guarantees. + type Layer = { readonly logMismatches?: unknown; readonly mismatchLogging?: { readonly value?: unknown; readonly diff?: unknown } }; + const layerLeaves = (shadow: Layer | undefined): { value?: unknown; diff?: unknown } => { + const leaves: { value?: unknown; diff?: unknown } = {}; + if (shadow?.logMismatches !== undefined) { + leaves.value = shadow.logMismatches === true; + leaves.diff = shadow.logMismatches === true ? "values" : "off"; + } + if (shadow?.mismatchLogging?.value !== undefined) { + leaves.value = shadow.mismatchLogging.value; + } + if (shadow?.mismatchLogging?.diff !== undefined) { + leaves.diff = shadow.mismatchLogging.diff; + } + return leaves; + }; + const expectedLeaves = (defaultShadow: Layer | undefined, runtimeShadow: Layer | undefined) => { + const inherited = layerLeaves(defaultShadow); + const written = layerLeaves(runtimeShadow); + const value = written.value !== undefined ? written.value : inherited.value; + const diff = written.diff !== undefined ? written.diff : inherited.diff; + const validValue = value === undefined || typeof value === "boolean"; + const validDiff = diff === undefined || diff === "off" || diff === "paths" || diff === "values"; + const malformedShorthand = runtimeShadow?.logMismatches !== undefined + && typeof runtimeShadow.logMismatches !== "boolean"; + return { + value: validValue && value === true, + diff: validDiff && diff !== undefined ? diff : "off", + invalid: malformedShorthand || !validValue || !validDiff, + }; + }; + const ABSENT = Symbol("absent"); + const OWN_UNDEFINED = Symbol("own undefined"); + // Builds one shadow group, with each field absent, written as undefined, + // or written with the given value. + const layer = (shorthand: unknown, group: unknown): Record => { + const shadow: Record = {}; + for (const [name, field] of [["logMismatches", shorthand], ["mismatchLogging", group]] as const) { + if (field !== ABSENT) { + shadow[name] = field === OWN_UNDEFINED ? undefined : field; + } + } + return shadow; + }; + const defaultShorthands: unknown[] = [ABSENT, OWN_UNDEFINED, true, false]; + const defaultGroups: unknown[] = [ + ABSENT, + OWN_UNDEFINED, + {}, + { value: true }, + { value: false }, + { diff: "paths" }, + { diff: "off" }, + { value: true, diff: "values" }, + { value: undefined, diff: "paths" }, + ]; + const runtimeShorthands: unknown[] = [ABSENT, OWN_UNDEFINED, true, false, "yes", null, 0]; + const runtimeGroups: unknown[] = [ + ABSENT, + OWN_UNDEFINED, + { value: true }, + { value: false }, + { diff: "paths" }, + { diff: "values" }, + { value: true, diff: "off" }, + { value: "yes" }, + { value: null }, + { diff: "full" }, + { diff: "toString" }, + { value: undefined }, + ]; + + let combinations = 0; + for (const defaultShorthand of defaultShorthands) { + for (const defaultGroup of defaultGroups) { + const defaultShadow = layer(defaultShorthand, defaultGroup); + const defaultConfig = new DialCacheKeyConfig({ shadow: defaultShadow as ShadowConfig }); + for (const runtimeShorthand of runtimeShorthands) { + for (const runtimeGroup of runtimeGroups) { + const runtimeShadow = layer(runtimeShorthand, runtimeGroup); + const expected = expectedLeaves(defaultShadow, runtimeShadow); + // A constructed runtime config clones the group; a raw provider + // result reaches the merge as written. + for (const runtime of [ + new DialCacheKeyConfig({ shadow: runtimeShadow as ShadowConfig }), + { ttlSec: {}, ramp: {}, shadow: runtimeShadow } as unknown as DialCacheKeyConfig, + ]) { + const merged = await fetchKeyConfig(async () => runtime, key(defaultConfig)); + expect(resolveMismatchLogging(merged!.shadow!)).toEqual(expected); + combinations += 1; + } + } + } + } + } + expect(combinations).toBe(4 * 9 * 7 * 12 * 2); + + // disabled() on either side, and a runtime config without a shadow group. + const disabledShadow = DialCacheKeyConfig.disabled().shadow!; + for (const [defaultShadow, runtime] of [ + [{ logMismatches: true }, DialCacheKeyConfig.disabled()], + [disabledShadow, new DialCacheKeyConfig({ shadow: { mismatchLogging: { diff: "paths" } } })], + [{ logMismatches: true, mismatchLogging: { diff: "paths" } }, new DialCacheKeyConfig({ ttlSec: {} })], + ] as const) { + const merged = await fetchKeyConfig( + async () => runtime, + key(new DialCacheKeyConfig({ shadow: defaultShadow })), + ); + expect(resolveMismatchLogging(merged!.shadow!)).toEqual(expectedLeaves(defaultShadow, runtime.shadow)); + } + }); + + it("expands the logMismatches shorthand before the same group's leaves override it", () => { + expect(expandMismatchLogging({})).toStrictEqual({}); + expect(expandMismatchLogging({ logMismatches: true })).toStrictEqual({ value: true, diff: "values" }); + expect(expandMismatchLogging({ logMismatches: false })).toStrictEqual({ value: false, diff: "off" }); + expect(expandMismatchLogging(rawShadow({ logMismatches: "yes" }))).toStrictEqual({ value: false, diff: "off", invalid: true }); + expect(expandMismatchLogging({ logMismatches: true, mismatchLogging: { value: false } })) + .toStrictEqual({ value: false, diff: "values" }); + expect(expandMismatchLogging(rawShadow({ logMismatches: "yes", mismatchLogging: { value: true, diff: "paths" } }))) + .toStrictEqual({ value: true, diff: "paths", invalid: true }); + // An undefined leaf is unwritten, and written leaves pass through + // unvalidated: resolution owns their validation. + expect(expandMismatchLogging(rawShadow({ logMismatches: true, mismatchLogging: { value: undefined } }))) + .toStrictEqual({ value: true, diff: "values" }); + expect(expandMismatchLogging(rawShadow({ mismatchLogging: { diff: "full" } }))).toStrictEqual({ diff: "full" }); + }); + + it.each([ + ["null", null], + ["an array", []], + ["a number", 1], + ["a string", "paths"], + ])("rejects a provider result whose mismatchLogging is %s", async (_name, mismatchLogging) => { + const runtime = { ttlSec: {}, ramp: {}, shadow: { mismatchLogging } } as unknown as DialCacheKeyConfig; + + await expect(fetchKeyConfig(async () => runtime, key())).rejects.toThrow( + new TypeError("DialCache shadow mismatchLogging config must be an object"), + ); + }); + it("keeps invalid stale-on-error policy diagnostic-only in remote resolution", () => { const remoteKey = key(); diff --git a/typescript/test/dialcache-shadow-confirmation.test.ts b/typescript/test/dialcache-shadow-confirmation.test.ts index 28db2d66..3dccb8ab 100644 --- a/typescript/test/dialcache-shadow-confirmation.test.ts +++ b/typescript/test/dialcache-shadow-confirmation.test.ts @@ -8,6 +8,7 @@ import { DialCacheKey, DialCacheKeyConfig, FallbackTimeoutError, + type CacheConfigProvider, type CacheMetricLabels, type CoalescedMetricLabels, type DialCacheConfig, @@ -16,6 +17,7 @@ import { type DisabledMetricLabels, type ErrorMetricLabels, type InvalidationMetricLabels, + type Logger, type RedisCachePayload, type RedisInvalidationRequest, type RedisReadContext, @@ -25,6 +27,11 @@ import { type RedisWriteRequest, type SerializationMetricLabels, type Serializer, + type ShadowMismatchDiff, + type ShadowMismatchEntry, + type ShadowMismatchLoggingConfig, + type ShadowMismatchLoggingMetricLabels, + type ShadowMismatchLoggingOutcome, type ShadowValidationMetricLabels, } from "../src/index.js"; import { @@ -32,9 +39,11 @@ import { deterministicShadowRampSample, } from "../src/internal/ramp.js"; import { + SHADOW_LOG_DIFF_MAX_BYTES, SHADOW_LOG_KEY_MAX_BYTES, SHADOW_LOG_TRUNCATION_MARKER, SHADOW_LOG_VALUE_MAX_BYTES, + SHADOW_MISMATCH_SIDE_MAX_BYTES, } from "../src/internal/shadow-log-json.js"; import { REMOTE_SHADOW_CACHE_LAYER } from "../src/metrics.js"; @@ -124,6 +133,7 @@ class RecordingMetrics implements DialCacheMetricsAdapter { readonly ordinaryEvents: OrdinaryMetricEvent[] = []; readonly shadowEvents: ShadowValidationMetricLabels[] = []; readonly shadowAgeEvents: ShadowAgeEvent[] = []; + readonly shadowMismatchLoggingEvents: ShadowMismatchLoggingMetricLabels[] = []; readonly futureTimestampEvents: FutureTimestampEvent[] = []; request(labels: CacheMetricLabels): void { @@ -158,6 +168,10 @@ class RecordingMetrics implements DialCacheMetricsAdapter { this.shadowAgeEvents.push({ labels: { ...labels }, seconds }); } + shadowMismatchLogging(labels: ShadowMismatchLoggingMetricLabels): void { + this.shadowMismatchLoggingEvents.push({ ...labels }); + } + observeFutureTimestampOffset(labels: CacheMetricLabels, seconds: number): void { this.futureTimestampEvents.push({ labels: { ...labels }, seconds }); } @@ -188,6 +202,7 @@ function remoteConfig( shadowPercentage = 100, logging: { readonly logMismatches?: boolean; + readonly mismatchLogging?: ShadowMismatchLoggingConfig; } = {}, ): DialCacheKeyConfig { return new DialCacheKeyConfig({ @@ -254,6 +269,124 @@ async function waitForShadowEvents(metrics: RecordingMetrics, count: number): Pr }, { timeout: 2_000, interval: 1 }); } +async function waitForMismatchLoggingEvents(metrics: RecordingMetrics, count: number): Promise { + await vi.waitFor(() => { + expect(metrics.shadowMismatchLoggingEvents).toHaveLength(count); + }, { timeout: 2_000, interval: 1 }); +} + +async function expectNoUnhandledRejection(run: () => Promise): Promise { + const unhandled = vi.fn(); + process.on("unhandledRejection", unhandled); + try { + await run(); + await nextImmediate(); + } finally { + process.off("unhandledRejection", unhandled); + } + expect(unhandled).not.toHaveBeenCalled(); +} + +interface MismatchRecord { + readonly id: string; + readonly version: number; + readonly tags: readonly string[]; + readonly extra?: boolean; +} + +const mismatchCachedRecord: MismatchRecord = { id: "123", version: 1, tags: ["a"] }; +const mismatchSourceRecord: MismatchRecord = { id: "123", version: 2, tags: ["a", "b"], extra: true }; +const MISMATCH_PATH_FIELDS = { + mismatchPaths: ["/extra", "/tags", "/version"], + mismatchPathCount: 3, +} as const; +const MISMATCH_DIFF_JSON = '[{"path":"/extra","source":true},' + + '{"path":"/tags","cached":["a"],"source":["a","b"]},' + + '{"path":"/version","cached":1,"source":2}]'; +const MISMATCH_VALUE_FIELDS = { + cachedValueJson: '{"id":"123","version":1,"tags":["a"]}', + sourceValueJson: '{"id":"123","version":2,"tags":["a","b"],"extra":true}', +} as const; + +interface ConfirmedMismatchOptions { + readonly warn: Logger["warn"]; + readonly metrics?: RecordingMetrics; + readonly cacheConfigProvider?: CacheConfigProvider; + readonly shadowMismatchDiff?: ShadowMismatchDiff; + readonly api?: "cached" | "getOrLoad"; +} + +/** + * Serves `mismatchCachedRecord` from Redis under key "123", loads + * `mismatchSourceRecord` from the source, confirms the unchanged C1, and + * waits for the verdict. + */ +async function runConfirmedMismatch( + useCase: string, + defaultConfig: DialCacheKeyConfig, + options: ConfirmedMismatchOptions, +): Promise<{ readonly metrics: RecordingMetrics; readonly result: MismatchRecord }> { + const payload = JSON.stringify(mismatchCachedRecord); + const redis = new ScriptedRedis([() => payload, () => payload]); + const metrics = options.metrics ?? new RecordingMetrics(); + const dialcache = createCache(redis, metrics, { + ...(options.cacheConfigProvider === undefined ? {} : { cacheConfigProvider: options.cacheConfigProvider }), + logger: { + debug: () => undefined, + error: () => undefined, + warn: options.warn, + }, + }); + const load = async (): Promise => mismatchSourceRecord; + const hook = options.shadowMismatchDiff === undefined ? {} : { shadowMismatchDiff: options.shadowMismatchDiff }; + const result = await dialcache.enable(async () => options.api === "getOrLoad" + ? await dialcache.getOrLoad(load, { ...trackedOptions(useCase, defaultConfig), key: "123", ...hook }) + : await dialcache.cached(load, { ...trackedOptions(useCase, defaultConfig), cacheKey: () => "123", ...hook })()); + await waitForShadowEvents(metrics, 1); + return { metrics, result }; +} + +/** The always-present warning fields for key "123", with the verdict's observed value age. */ +function mismatchWarning( + useCase: string, + metrics: RecordingMetrics, + fields: Readonly> = {}, +): Record { + expect(metrics.shadowAgeEvents).toHaveLength(1); + return { + cacheNamespace: "urn", + useCase, + keyType: "user_id", + outcome: "mismatch", + cachedValueAgeSeconds: metrics.shadowAgeEvents[0]!.seconds, + cacheKey: `{urn:user_id:123}#${useCase}`, + ...fields, + }; +} + +interface BigintRecord { + readonly id: string; + readonly version: number | bigint; +} + +const bigintRecordSerializer: Serializer = { + dump: (value) => JSON.stringify({ ...value, version: Number(value.version) }), + load: (value) => JSON.parse(Buffer.isBuffer(value) ? value.toString("utf8") : value) as BigintRecord, +}; + +function mismatchLoggingEvent( + useCase: string, + outcome: ShadowMismatchLoggingOutcome, +): ShadowMismatchLoggingMetricLabels { + return { cacheNamespace: "urn", useCase, keyType: "user_id", outcome }; +} + +function configResolutionErrors(metrics: RecordingMetrics): Array> { + return metrics.ordinaryEvents + .filter(({ name, labels }) => name === "error" && labels.error === "config_resolution") + .map(({ labels }) => labels); +} + function expectTrackedReads( redis: ScriptedRedis, count: number, @@ -375,8 +508,11 @@ describe("DialCache Redis shadow confirmation", () => { await dialcache.enable(async () => await getUser()); await waitForShadowEvents(metrics, 1); + await nextImmediate(); + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); expect(warn).not.toHaveBeenCalled(); + expect(metrics.shadowMismatchLoggingEvents).toEqual([]); }); it("logs one bounded JSON warning for a served-hit confirmed mismatch", async () => { @@ -412,7 +548,9 @@ describe("DialCache Redis shadow confirmation", () => { await dialcache.enable(async () => await getUser()); await waitForShadowEvents(metrics, 1); + await waitForMismatchLoggingEvents(metrics, 1); + expect(metrics.shadowAgeEvents).toHaveLength(1); expect(warn).toHaveBeenCalledTimes(1); expect(warn).toHaveBeenCalledWith( "DialCache shadow validation mismatch", @@ -421,11 +559,38 @@ describe("DialCache Redis shadow confirmation", () => { useCase: "ShadowMismatchJson", keyType: "user_id", outcome: "mismatch", + cachedValueAgeSeconds: metrics.shadowAgeEvents[0]!.seconds, cacheKey: "{private-namespace:user_id:private-id}?tenant=private-tenant#ShadowMismatchJson", + mismatchPaths: ["/version"], + mismatchPathCount: 1, + diffJson: '[{"path":"/version","cached":1,"source":2}]', cachedValueJson: '{"id":"private-id","version":1}', sourceValueJson: '{"id":"private-id","version":2}', }, ); + expect(Object.keys((warn.mock.calls[0] as unknown as [string, object])[1])).toEqual([ + "cacheNamespace", + "useCase", + "keyType", + "outcome", + "cachedValueAgeSeconds", + "cacheKey", + "mismatchPaths", + "mismatchPathCount", + "diffJson", + "cachedValueJson", + "sourceValueJson", + ]); + // Labels come from the key, not from the mutated verdict labels, and a + // rejected logger promise counts as logger_error. + expect(metrics.shadowMismatchLoggingEvents).toEqual([ + { + cacheNamespace: "private-namespace", + useCase: "ShadowMismatchJson", + keyType: "user_id", + outcome: "logger_error", + }, + ]); await nextImmediate(); }); @@ -465,7 +630,9 @@ describe("DialCache Redis shadow confirmation", () => { await expect(dialcache.enable(async () => await getUser())).resolves.toBe(sourceValue); await waitForShadowEvents(metrics, 1); + await waitForMismatchLoggingEvents(metrics, 1); + expect(metrics.shadowAgeEvents).toHaveLength(1); expect(warn).toHaveBeenCalledTimes(1); expect(warn).toHaveBeenCalledWith( "DialCache shadow validation mismatch", @@ -474,15 +641,22 @@ describe("DialCache Redis shadow confirmation", () => { useCase: "ShadowMismatchRampedDownJson", keyType: "user_id", outcome: "mismatch", + cachedValueAgeSeconds: metrics.shadowAgeEvents[0]!.seconds, cacheKey: "{urn:user_id:123}?locale=en-US#ShadowMismatchRampedDownJson", + mismatchPaths: ["/version"], + mismatchPathCount: 1, + diffJson: '[{"path":"/version","cached":1,"source":2}]', cachedValueJson: '{"id":"123","version":1}', sourceValueJson: '{"id":"123","version":2}', }, ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([ + mismatchLoggingEvent("ShadowMismatchRampedDownJson", "logged"), + ]); expect(serializer.dump).not.toHaveBeenCalled(); }); - it("falls back to a metadata warning when JSON detail construction throws", async () => { + it("logs every requested field as null with render_error when warning construction throws", async () => { const cachedValue = { id: "123", version: 1 }; const sourceValue = { id: "123", version: 2 }; const payload = JSON.stringify(cachedValue); @@ -519,8 +693,10 @@ describe("DialCache Redis shadow confirmation", () => { try { sourceGate.resolve(sourceValue); await waitForShadowEvents(metrics, 1); + await waitForMismatchLoggingEvents(metrics, 1); expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); + expect(metrics.shadowAgeEvents).toHaveLength(1); expect(warn).toHaveBeenCalledOnce(); expect(warn).toHaveBeenCalledWith( "DialCache shadow validation mismatch", @@ -529,8 +705,19 @@ describe("DialCache Redis shadow confirmation", () => { useCase: "ShadowMismatchJsonFailure", keyType: "user_id", outcome: "mismatch", + cachedValueAgeSeconds: metrics.shadowAgeEvents[0]!.seconds, + mismatchPaths: null, + mismatchPathCount: null, + diffJson: null, + cachedValueJson: null, + sourceValueJson: null, + loggingOutcome: "render_error", }, ); + expect(warn.mock.calls[0]?.[1]).not.toHaveProperty("cacheKey"); + expect(metrics.shadowMismatchLoggingEvents).toEqual([ + mismatchLoggingEvent("ShadowMismatchJsonFailure", "render_error"), + ]); } finally { sourceGate.resolve(sourceValue); encodeInto.mockRestore(); @@ -564,12 +751,15 @@ describe("DialCache Redis shadow confirmation", () => { await dialcache.enable(async () => await getUser()); await waitForShadowEvents(metrics, 1); + await waitForMismatchLoggingEvents(metrics, 1); expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); expect(warn).toHaveBeenCalledTimes(1); const warning = warn.mock.calls[0]?.[1] as Record; expect(warning).not.toHaveProperty("cachedValue"); expect(warning).not.toHaveProperty("sourceValue"); + expect(warning).not.toHaveProperty("loggingOutcome"); + expect(warning.cachedValueAgeSeconds).toBe(metrics.shadowAgeEvents[0]?.seconds); for (const [field, maxBytes] of [ ["cacheKey", SHADOW_LOG_KEY_MAX_BYTES], ["cachedValueJson", SHADOW_LOG_VALUE_MAX_BYTES], @@ -581,6 +771,701 @@ describe("DialCache Redis shadow confirmation", () => { expect((preview as string).endsWith(SHADOW_LOG_TRUNCATION_MARKER)).toBe(true); expect(preview).not.toContain("\uFFFD"); } + + // The ids are equal, so the one differing member carries both sides, + // each clamped to its JSON text. + expect(warning.mismatchPaths).toEqual(["/content"]); + expect(warning.mismatchPathCount).toBe(1); + expect(typeof warning.diffJson).toBe("string"); + expect(Buffer.byteLength(warning.diffJson as string)).toBeLessThanOrEqual(SHADOW_LOG_DIFF_MAX_BYTES); + const entries = JSON.parse(warning.diffJson as string) as Array>; + expect(entries.map((entry) => Object.keys(entry))).toEqual([["path", "cached", "source"]]); + for (const side of [entries[0]!.cached, entries[0]!.source]) { + expect(typeof side).toBe("string"); + expect(Buffer.byteLength(side as string)).toBeLessThanOrEqual(SHADOW_MISMATCH_SIDE_MAX_BYTES); + expect((side as string).startsWith('"')).toBe(true); + expect((side as string).endsWith(SHADOW_LOG_TRUNCATION_MARKER)).toBe(true); + expect(side).not.toContain("\uFFFD"); + } + expect(metrics.shadowMismatchLoggingEvents).toEqual([ + mismatchLoggingEvent("ShadowMismatchClampedJson", "logged"), + ]); + }); + + it("logs only the paths and their count at diff paths", async () => { + const useCase = "ShadowMismatchPathsOnly"; + const warn = vi.fn(); + const { metrics } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, { mismatchLogging: { diff: "paths" } }), + { warn }, + ); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, MISMATCH_PATH_FIELDS), + ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "logged")]); + }); + + it("logs the paths and diffJson without whole-value JSON at diff values with value false", async () => { + const useCase = "ShadowMismatchValuesDiffOnly"; + const warn = vi.fn(); + const { metrics } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, { mismatchLogging: { diff: "values", value: false } }), + { warn }, + ); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, { ...MISMATCH_PATH_FIELDS, diffJson: MISMATCH_DIFF_JSON }), + ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "logged")]); + }); + + it("logs only the whole-value JSON and never runs the diff hook at value true alone", async () => { + const useCase = "ShadowMismatchValueOnly"; + const warn = vi.fn(); + const shadowMismatchDiff = vi.fn((): readonly ShadowMismatchEntry[] => [{ path: "/hook" }]); + const { metrics } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, { mismatchLogging: { value: true } }), + { warn, shadowMismatchDiff }, + ); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, MISMATCH_VALUE_FIELDS), + ); + expect(shadowMismatchDiff).not.toHaveBeenCalled(); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "logged")]); + }); + + it("logs the exact value age the verdict observed as cachedValueAgeSeconds", async () => { + const nowMs = 1_700_000_090_000; + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(nowMs); + try { + const useCase = "ShadowMismatchWarningAge"; + const payload = JSON.stringify(mismatchCachedRecord); + const redis = new ScriptedRedis([() => payload, () => payload]); + redis.frameCreatedAtMs = nowMs - 30_500; + const metrics = new RecordingMetrics(); + const warn = vi.fn(); + const dialcache = createCache(redis, metrics, { + logger: { + debug: () => undefined, + error: () => undefined, + warn, + }, + }); + const getUser = dialcache.cached(async () => mismatchSourceRecord, { + ...trackedOptions(useCase, remoteConfig(100, 100, { mismatchLogging: { diff: "paths" } })), + cacheKey: () => "123", + }); + + await dialcache.enable(async () => await getUser()); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(metrics.shadowAgeEvents.map(({ seconds }) => seconds)).toEqual([30.5]); + expect(warn).toHaveBeenCalledOnce(); + expect(warn.mock.calls[0]?.[1]).toMatchObject({ cachedValueAgeSeconds: 30.5 }); + } finally { + nowSpy.mockRestore(); + } + }); + + it.each([ + { + diff: "paths" as const, + fields: { + mismatchPaths: ["/plan", "/profile/locale", "/profile/timezone", "/roles"], + mismatchPathCount: 4, + }, + }, + { + diff: "values" as const, + fields: { + mismatchPaths: ["/plan", "/profile/locale", "/profile/timezone", "/roles"], + mismatchPathCount: 4, + diffJson: '[{"path":"/plan","cached":"free","source":"pro"},' + + '{"path":"/profile/locale","cached":"en-US","source":"en-GB"},' + + '{"path":"/profile/timezone","source":"Europe/London"},' + + '{"path":"/roles","cached":["viewer"],"source":["viewer","editor"]}]', + }, + }, + ])("logs the documented GetUser example at diff $diff", async ({ diff, fields }) => { + // docs/shadow-validation.md shows these exact warnings; keep them in step. + const cached = { + id: "u_123", + email: "ana@example.com", + plan: "free", + roles: ["viewer"], + profile: { name: "Ana", locale: "en-US" }, + }; + const source = { + id: "u_123", + email: "ana@example.com", + plan: "pro", + roles: ["viewer", "editor"], + profile: { name: "Ana", locale: "en-GB", timezone: "Europe/London" }, + }; + const payload = JSON.stringify(cached); + const redis = new ScriptedRedis([() => payload, () => payload]); + const metrics = new RecordingMetrics(); + const warn = vi.fn(); + const dialcache = createCache(redis, metrics, { + namespace: "users-api", + logger: { debug: () => undefined, error: () => undefined, warn }, + }); + const getUser = dialcache.cached(async (_userId: string) => source, { + keyType: "user_id", + useCase: "GetUser", + cacheKey: (userId) => userId, + trackForInvalidation: true, + defaultConfig: remoteConfig(100, 100, { mismatchLogging: { diff } }), + }); + + await expect(dialcache.enable(async () => await getUser("u_123"))).resolves.toEqual(cached); + await waitForMismatchLoggingEvents(metrics, 1); + + const warning = { + cacheNamespace: "users-api", + useCase: "GetUser", + keyType: "user_id", + outcome: "mismatch", + cachedValueAgeSeconds: metrics.shadowAgeEvents[0]!.seconds, + cacheKey: "{users-api:user_id:u_123}#GetUser", + ...fields, + }; + expect(warn).toHaveBeenCalledExactlyOnceWith("DialCache shadow validation mismatch", warning); + // The page shows the fields in this order. + expect(Object.keys(warn.mock.calls[0]![1] as object)).toEqual(Object.keys(warning)); + }); + + it("emits one logged event per confirmed mismatch after its verdict and none for a match", async () => { + const useCase = "ShadowMismatchLoggingOrder"; + const matching = JSON.stringify({ id: "a", version: 1 }); + const redis = new ScriptedRedis([ + () => matching, + () => JSON.stringify({ id: "b", version: 1 }), + () => JSON.stringify({ id: "b", version: 1 }), + () => JSON.stringify({ id: "c", version: 1 }), + () => JSON.stringify({ id: "c", version: 1 }), + ]); + const metrics = new RecordingMetrics(); + const shadowValidation = vi.spyOn(metrics, "shadowValidation"); + const observeShadowValueAge = vi.spyOn(metrics, "observeShadowValueAge"); + const shadowMismatchLogging = vi.spyOn(metrics, "shadowMismatchLogging"); + const warn = vi.fn(); + const dialcache = createCache(redis, metrics, { + logger: { + debug: () => undefined, + error: () => undefined, + warn, + }, + }); + const getUser = dialcache.cached(async (id: string) => ({ id, version: id === "a" ? 1 : 2 }), { + ...trackedOptions(useCase, remoteConfig(100, 100, { mismatchLogging: { diff: "paths" } })), + cacheKey: (id) => id, + }); + + await dialcache.enable(async () => await getUser("a")); + await waitForShadowEvents(metrics, 1); + await nextImmediate(); + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["match"]); + expect(metrics.shadowMismatchLoggingEvents).toEqual([]); + + await dialcache.enable(async () => await getUser("b")); + await waitForMismatchLoggingEvents(metrics, 1); + await dialcache.enable(async () => await getUser("c")); + await waitForMismatchLoggingEvents(metrics, 2); + await nextImmediate(); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["match", "mismatch", "mismatch"]); + expect(metrics.shadowMismatchLoggingEvents).toEqual([ + mismatchLoggingEvent(useCase, "logged"), + mismatchLoggingEvent(useCase, "logged"), + ]); + expect(warn).toHaveBeenCalledTimes(2); + // The match verdict observes its value age too, so both verdict streams + // are offset by one against the warning streams. + expect(observeShadowValueAge).toHaveBeenCalledTimes(3); + for (const attempt of [0, 1]) { + const sequence = [ + shadowValidation.mock.invocationCallOrder[attempt + 1]!, + observeShadowValueAge.mock.invocationCallOrder[attempt + 1]!, + warn.mock.invocationCallOrder[attempt]!, + shadowMismatchLogging.mock.invocationCallOrder[attempt]!, + ]; + expect(sequence).toEqual([...sequence].sort((left, right) => left - right)); + } + }); + + it.each([ + ["null", null], + ["a string", "paths"], + ["an object", { path: "/version" }], + ])("rejects %s as shadowMismatchDiff where each API captures it", async (_label, hook) => { + const dialcache = new DialCache(); + const invalidHook = hook as unknown as ShadowMismatchDiff; + const source = async (): Promise => mismatchSourceRecord; + const getOrLoadOptions = { + keyType: "user_id", + useCase: "InvalidGetOrLoadShadowMismatchDiff", + key: "123", + shadowMismatchDiff: invalidHook, + } as const; + + expect(() => + dialcache.cached(source, { + keyType: "user_id", + useCase: "InvalidCachedShadowMismatchDiff", + cacheKey: () => "123", + shadowMismatchDiff: invalidHook, + }), + ).toThrow(new TypeError("DialCache shadowMismatchDiff must be a function")); + // The rejected registration leaves the use case free for a valid one. + expect(() => + dialcache.cached(source, { + keyType: "user_id", + useCase: "InvalidCachedShadowMismatchDiff", + cacheKey: () => "123", + shadowMismatchDiff: () => [], + }), + ).not.toThrow(); + expect(() => dialcache.getOrLoad(source, getOrLoadOptions)).toThrow(TypeError); + await expect(dialcache.enable(async () => await dialcache.getOrLoad(source, getOrLoadOptions))) + .rejects.toThrow(TypeError); + }); + + it.each(["cached", "getOrLoad"] as const)( + "logs shadowMismatchDiff entries in place of the built-in walk through %s()", + async (api) => { + const useCase = `ShadowMismatchDiffHook${api}`; + const warn = vi.fn(); + const shadowMismatchDiff = vi.fn((): readonly ShadowMismatchEntry[] => [ + { path: "/custom", cached: "c", source: "s" }, + { path: "/only-path" }, + { path: "/one-side", source: { nested: true } }, + ]); + const { metrics, result } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, { logMismatches: true }), + { warn, shadowMismatchDiff, api }, + ); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(result).toEqual(mismatchCachedRecord); + expect(shadowMismatchDiff).toHaveBeenCalledOnce(); + expect(shadowMismatchDiff).toHaveBeenCalledWith(mismatchCachedRecord, mismatchSourceRecord); + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, { + mismatchPaths: ["/custom", "/only-path", "/one-side"], + mismatchPathCount: 3, + diffJson: '[{"path":"/custom","cached":"c","source":"s"},' + + '{"path":"/only-path"},' + + '{"path":"/one-side","source":{"nested":true}}]', + ...MISMATCH_VALUE_FIELDS, + }), + ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "logged")]); + }, + ); + + it.each([ + { + name: "throws", + useCase: "ShadowMismatchDiffHookThrows", + hook: (): readonly ShadowMismatchEntry[] => { + throw new Error("diff hook unavailable"); + }, + }, + { + name: "returns a rejecting promise", + useCase: "ShadowMismatchDiffHookPromise", + hook: (): readonly ShadowMismatchEntry[] => + Promise.reject(new Error("async diff hooks are unsupported")) as unknown as readonly ShadowMismatchEntry[], + }, + ])("logs null diff fields and hook_error without touching the verdict when the hook $name", async ({ + useCase, + hook, + }) => { + const warn = vi.fn(); + await expectNoUnhandledRejection(async () => { + const { metrics, result } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, { logMismatches: true }), + { warn, shadowMismatchDiff: hook }, + ); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(result).toEqual(mismatchCachedRecord); + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, { + mismatchPaths: null, + mismatchPathCount: null, + diffJson: null, + ...MISMATCH_VALUE_FIELDS, + loggingOutcome: "hook_error", + }), + ); + expect(Object.keys(warn.mock.calls[0]![1] as object).at(-1)).toBe("loggingOutcome"); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "hook_error")]); + }); + }); + + it.each([ + { name: "the default deep equality", useCase: "ShadowMismatchBigintDefaultEquality", comparator: undefined }, + { + name: "a comparator", + useCase: "ShadowMismatchBigintComparator", + comparator: (cached: BigintRecord, source: BigintRecord) => cached.version === source.version, + }, + ])("logs render_error with null source and diff fields for a bigint source value under $name", async ({ + useCase, + comparator, + }) => { + const payload = JSON.stringify({ id: "123", version: 1 }); + const redis = new ScriptedRedis([() => payload, () => payload]); + const metrics = new RecordingMetrics(); + const warn = vi.fn(); + const dialcache = createCache(redis, metrics, { + logger: { + debug: () => undefined, + error: () => undefined, + warn, + }, + }); + const getUser = dialcache.cached(async (): Promise => ({ id: "123", version: 1n }), { + ...trackedOptions(useCase, remoteConfig(100, 100, { logMismatches: true })), + cacheKey: () => "123", + serializer: bigintRecordSerializer, + ...(comparator === undefined ? {} : { shadowComparator: comparator }), + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123", version: 1 }); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, { + mismatchPaths: null, + mismatchPathCount: null, + diffJson: null, + cachedValueJson: '{"id":"123","version":1}', + sourceValueJson: null, + loggingOutcome: "render_error", + }), + ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "render_error")]); + }); + + it("logs a cached undefined negative-cache entry as an absent side, not a render failure", async () => { + const useCase = "ShadowMismatchAbsentCachedValue"; + // The default JSON serializer's stored form of a top-level undefined. + const payload = "__dialcache_json_undefined_v1__"; + const redis = new ScriptedRedis([() => payload, () => payload]); + const metrics = new RecordingMetrics(); + const warn = vi.fn(); + const dialcache = createCache(redis, metrics, { + logger: { + debug: () => undefined, + error: () => undefined, + warn, + }, + }); + const getUser = dialcache.cached(async (): Promise<{ id: string } | undefined> => ({ id: "123" }), { + ...trackedOptions(useCase, remoteConfig(100, 100, { logMismatches: true })), + cacheKey: () => "123", + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toBeUndefined(); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, { + mismatchPaths: [""], + mismatchPathCount: 1, + diffJson: '[{"path":"","source":{"id":"123"}}]', + cachedValueJson: null, + sourceValueJson: '{"id":"123"}', + }), + ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "logged")]); + }); + + it("passes an absent cached side to shadowMismatchDiff as the raw undefined", async () => { + const useCase = "ShadowMismatchAbsentCachedHookInput"; + // The default JSON serializer's stored form of a top-level undefined. + const payload = "__dialcache_json_undefined_v1__"; + const redis = new ScriptedRedis([() => payload, () => payload]); + const metrics = new RecordingMetrics(); + const warn = vi.fn(); + const dialcache = createCache(redis, metrics, { + logger: { + debug: () => undefined, + error: () => undefined, + warn, + }, + }); + const hookInputs: unknown[][] = []; + const getUser = dialcache.cached(async (): Promise<{ id: string } | undefined> => ({ id: "123" }), { + ...trackedOptions(useCase, remoteConfig(100, 100, { mismatchLogging: { diff: "values" } })), + cacheKey: () => "123", + shadowMismatchDiff: (cached, source) => { + hookInputs.push([cached, source]); + return [{ path: "", source }]; + }, + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toBeUndefined(); + await waitForMismatchLoggingEvents(metrics, 1); + + // D13: the hook receives the raw values, so an absent side stays + // undefined rather than becoming JSON null. + expect(hookInputs).toStrictEqual([[undefined, { id: "123" }]]); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, { + mismatchPaths: [""], + mismatchPathCount: 1, + diffJson: '[{"path":"","source":{"id":"123"}}]', + }), + ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "logged")]); + }); + + it("logs hook paths without rendering values at diff paths, even for a bigint source value", async () => { + const useCase = "ShadowMismatchBigintHookPaths"; + const payload = JSON.stringify({ id: "123", version: 1 }); + const redis = new ScriptedRedis([() => payload, () => payload]); + const metrics = new RecordingMetrics(); + const warn = vi.fn(); + const dialcache = createCache(redis, metrics, { + logger: { + debug: () => undefined, + error: () => undefined, + warn, + }, + }); + const getUser = dialcache.cached(async (): Promise => ({ id: "123", version: 1n }), { + ...trackedOptions(useCase, remoteConfig(100, 100, { mismatchLogging: { diff: "paths" } })), + cacheKey: () => "123", + serializer: bigintRecordSerializer, + shadowMismatchDiff: () => [{ path: "/version" }], + }); + + await dialcache.enable(async () => await getUser()); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, { mismatchPaths: ["/version"], mismatchPathCount: 1 }), + ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "logged")]); + }); + + it.each<{ + readonly name: string; + readonly useCase: string; + readonly warn: () => unknown; + readonly outcome: ShadowMismatchLoggingOutcome; + }>([ + { + name: "throws synchronously", + useCase: "ShadowMismatchLoggerThrows", + warn: () => { + throw new Error("logger unavailable"); + }, + outcome: "logger_error", + }, + { + name: "returns a rejected promise", + useCase: "ShadowMismatchLoggerRejects", + warn: () => Promise.reject(new Error("logger transport unavailable")), + outcome: "logger_error", + }, + { + name: "returns a resolved promise", + useCase: "ShadowMismatchLoggerResolves", + warn: () => Promise.resolve(), + outcome: "logged", + }, + { + name: "returns a thenable that rejects", + useCase: "ShadowMismatchLoggerThenableRejects", + warn: () => ({ + then(_resolve: unknown, reject: (reason: unknown) => void): void { + reject(new Error("logger transport unavailable")); + }, + }), + outcome: "logger_error", + }, + ])("records $outcome and leaves the caller unaffected when the logger $name", async ({ + useCase, + warn: warnImplementation, + outcome, + }) => { + // A plain function rather than vi.fn: a spy attaches its own handlers to a + // returned promise, which would hide a rejection DialCache left unconsumed. + const calls: unknown[][] = []; + const warn = (...args: unknown[]): unknown => { + calls.push(args); + return warnImplementation(); + }; + await expectNoUnhandledRejection(async () => { + const { metrics, result } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, { logMismatches: true }), + { warn }, + ); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(result).toEqual(mismatchCachedRecord); + expect(metrics.shadowEvents.map(({ outcome: verdict }) => verdict)).toEqual(["mismatch"]); + expect(calls).toEqual([[ + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, { + ...MISMATCH_PATH_FIELDS, + diffJson: MISMATCH_DIFF_JSON, + ...MISMATCH_VALUE_FIELDS, + }), + ]]); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, outcome)]); + }); + }); + + it.each(([ + { + name: "the hook throws", + useCase: "ShadowMismatchHookAndLoggerThrow", + shadowMismatchDiff: () => { + throw new Error("hook failed"); + }, + fields: { mismatchPaths: null, mismatchPathCount: null, diffJson: null }, + outcome: "hook_error", + }, + { + name: "a hook side has no JSON rendering", + useCase: "ShadowMismatchRenderAndLoggerThrow", + shadowMismatchDiff: () => [{ path: "/version", cached: 1n }], + fields: { mismatchPaths: ["/version"], mismatchPathCount: 1, diffJson: null }, + outcome: "render_error", + }, + ] as const satisfies readonly { + readonly name: string; + readonly useCase: string; + readonly shadowMismatchDiff: ShadowMismatchDiff; + readonly fields: Record; + readonly outcome: ShadowMismatchLoggingOutcome; + }[]).flatMap((failure) => (["throws", "rejects"] as const).map((loggerFailure) => ({ + ...failure, + useCase: `${failure.useCase}${loggerFailure}`, + loggerFailure, + }))))("records $outcome rather than logger_error when $name and the logger $loggerFailure", async ({ + useCase, + shadowMismatchDiff, + fields, + outcome, + loggerFailure, + }) => { + const calls: unknown[][] = []; + const warn = (...args: unknown[]): unknown => { + calls.push(args); + if (loggerFailure === "rejects") { + return Promise.reject(new Error("logger transport unavailable")); + } + throw new Error("logger unavailable"); + }; + await expectNoUnhandledRejection(async () => { + const { metrics, result } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, { mismatchLogging: { diff: "values" } }), + { warn, shadowMismatchDiff }, + ); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(result).toEqual(mismatchCachedRecord); + expect(calls).toEqual([[ + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, { ...fields, loggingOutcome: outcome }), + ]]); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, outcome)]); + }); + }); + + it.each([ + { + name: "the hook throws", + useCase: "ShadowMismatchHookFailsLoggerPending", + shadowMismatchDiff: (): never => { + throw new Error("hook failed"); + }, + outcome: "hook_error", + }, + { + name: "a hook side has no JSON rendering", + useCase: "ShadowMismatchRenderFailsLoggerPending", + shadowMismatchDiff: () => [{ path: "/version", cached: 1n }], + outcome: "render_error", + }, + ] as const)("records $outcome at once when $name while the logger's promise is still pending", async ({ + useCase, + shadowMismatchDiff, + outcome, + }) => { + let warnings = 0; + const warn = (): Promise => { + warnings += 1; + return new Promise(() => undefined); + }; + const { metrics } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, { mismatchLogging: { diff: "values" } }), + { warn, shadowMismatchDiff: shadowMismatchDiff as ShadowMismatchDiff }, + ); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(warnings).toBe(1); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, outcome)]); + }); + + it("records no logging outcome while a logger's promise never settles", async () => { + let warnings = 0; + const warn = (): Promise => { + warnings += 1; + return new Promise(() => undefined); + }; + const { metrics } = await runConfirmedMismatch( + "ShadowMismatchLoggerPending", + remoteConfig(100, 100, { logMismatches: true }), + { warn }, + ); + for (let turn = 0; turn < 5; turn += 1) { + await nextImmediate(); + } + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); + expect(warnings).toBe(1); + expect(metrics.shadowMismatchLoggingEvents).toEqual([]); }); it.each([ @@ -855,8 +1740,11 @@ describe("DialCache Redis shadow confirmation", () => { await dialcache.enable(async () => await getUser()); await waitForShadowEvents(metrics, 1); + await nextImmediate(); + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["superseded"]); expect(warn).not.toHaveBeenCalled(); + expect(metrics.shadowMismatchLoggingEvents).toEqual([]); }); it.each([ @@ -2002,6 +2890,7 @@ describe("DialCache Redis shadow confirmation", () => { && labels.error === "config_resolution" )).toHaveLength(1); expect(warn).not.toHaveBeenCalled(); + expect(metrics.shadowMismatchLoggingEvents).toEqual([]); }); it("treats DialCacheKeyConfig.disabled() as a complete shadow kill switch", async () => { @@ -2024,6 +2913,180 @@ describe("DialCache Redis shadow confirmation", () => { expect(metrics.shadowEvents).toHaveLength(0); }); + it("keeps the valid default leaf and records one config_resolution for an invalid runtime leaf", async () => { + const useCase = "ShadowMismatchInvalidRuntimeLeaf"; + const warn = vi.fn(); + const { metrics } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, { mismatchLogging: { value: true } }), + { + warn, + cacheConfigProvider: async () => new DialCacheKeyConfig({ + shadow: { mismatchLogging: { diff: "everything" as never } }, + }), + }, + ); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(configResolutionErrors(metrics)).toEqual([ + { + cacheNamespace: "urn", + useCase, + keyType: "user_id", + layer: CacheLayer.REMOTE, + error: "config_resolution", + inFallback: false, + }, + ]); + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, MISMATCH_VALUE_FIELDS), + ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "logged")]); + }); + + it("records one config_resolution for a malformed runtime shorthand and logs the leaves beside it", async () => { + const useCase = "ShadowMismatchMalformedShorthandLeaves"; + const warn = vi.fn(); + const { metrics } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, { logMismatches: true }), + { + warn, + cacheConfigProvider: async () => new DialCacheKeyConfig({ + shadow: { + logMismatches: "yes" as never, + mismatchLogging: { value: false, diff: "paths" }, + }, + }), + }, + ); + await waitForMismatchLoggingEvents(metrics, 1); + + expect(configResolutionErrors(metrics)).toEqual([ + { + cacheNamespace: "urn", + useCase, + keyType: "user_id", + layer: CacheLayer.REMOTE, + error: "config_resolution", + inFallback: false, + }, + ]); + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, MISMATCH_PATH_FIELDS), + ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "logged")]); + }); + + it.each([ + { + name: "every field but whole-value JSON for logMismatches true beside value false", + useCase: "ShadowMismatchRowShorthandTrueValueFalse", + defaults: { logMismatches: true, mismatchLogging: { value: false } }, + runtime: undefined, + fields: { ...MISMATCH_PATH_FIELDS, diffJson: MISMATCH_DIFF_JSON }, + }, + { + name: "only the paths for logMismatches false beside diff paths", + useCase: "ShadowMismatchRowShorthandFalseDiffPaths", + defaults: { logMismatches: false, mismatchLogging: { diff: "paths" } }, + runtime: undefined, + fields: MISMATCH_PATH_FIELDS, + }, + { + name: "nothing for a runtime logMismatches false over a default diff paths", + useCase: "ShadowMismatchRowRuntimeFalseOverPaths", + defaults: { mismatchLogging: { diff: "paths" } }, + runtime: { logMismatches: false }, + fields: null, + }, + { + name: "every field for a runtime logMismatches true over a default value false", + useCase: "ShadowMismatchRowRuntimeTrueOverValueFalse", + defaults: { mismatchLogging: { value: false } }, + runtime: { logMismatches: true }, + fields: { ...MISMATCH_PATH_FIELDS, diffJson: MISMATCH_DIFF_JSON, ...MISMATCH_VALUE_FIELDS }, + }, + ] as const)("logs $name", async ({ useCase, defaults, runtime, fields }) => { + const warn = vi.fn(); + const { metrics } = await runConfirmedMismatch( + useCase, + remoteConfig(100, 100, defaults), + { + warn, + ...(runtime === undefined + ? {} + : { cacheConfigProvider: async () => new DialCacheKeyConfig({ shadow: runtime }) }), + }, + ); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); + expect(configResolutionErrors(metrics)).toEqual([]); + if (fields === null) { + await nextImmediate(); + expect(warn).not.toHaveBeenCalled(); + expect(metrics.shadowMismatchLoggingEvents).toEqual([]); + return; + } + await waitForMismatchLoggingEvents(metrics, 1); + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + "DialCache shadow validation mismatch", + mismatchWarning(useCase, metrics, fields), + ); + expect(metrics.shadowMismatchLoggingEvents).toEqual([mismatchLoggingEvent(useCase, "logged")]); + }); + + it("lets DialCacheKeyConfig.disabled() override every inherited mismatch-logging leaf", async () => { + const defaults = remoteConfig(100, 100, { + logMismatches: true, + mismatchLogging: { value: true, diff: "values" }, + }); + const disabledShadow = DialCacheKeyConfig.disabled().shadow; + expect(disabledShadow).toEqual({ ramp: 0, mismatchLogging: { value: false, diff: "off" } }); + + const killedRedis = new ScriptedRedis([]); + const killedMetrics = new RecordingMetrics(); + const killedWarn = vi.fn(); + const killedCache = createCache(killedRedis, killedMetrics, { + cacheConfigProvider: async () => DialCacheKeyConfig.disabled(), + logger: { + debug: () => undefined, + error: () => undefined, + warn: killedWarn, + }, + }); + const getUser = killedCache.cached(async () => mismatchSourceRecord, { + ...trackedOptions("ShadowMismatchDisabledOverlay", defaults), + cacheKey: () => "123", + }); + await expect(killedCache.enable(async () => await getUser())).resolves.toEqual(mismatchSourceRecord); + await nextImmediate(); + + expect(killedRedis.requests).toHaveLength(0); + expect(killedMetrics.shadowEvents).toHaveLength(0); + expect(killedWarn).not.toHaveBeenCalled(); + expect(killedMetrics.shadowMismatchLoggingEvents).toEqual([]); + + // With only the shadow ramp restored, the verdict still runs, but the + // overlay's leaves replace both inherited ones. + const warn = vi.fn(); + const { metrics } = await runConfirmedMismatch("ShadowMismatchDisabledLeaves", defaults, { + warn, + cacheConfigProvider: async () => new DialCacheKeyConfig({ shadow: { ...disabledShadow, ramp: 100 } }), + }); + await nextImmediate(); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); + expect(warn).not.toHaveBeenCalled(); + expect(metrics.shadowMismatchLoggingEvents).toEqual([]); + expect(configResolutionErrors(metrics)).toEqual([]); + }); + it("keeps partial Redis-serving and shadow-observation cohorts independent", async () => { const useCase = "ShadowIndependentPartialCohorts"; const keyFor = (id: string) => new DialCacheKey({ diff --git a/typescript/test/dialcache-stale-recovery-policy.test.ts b/typescript/test/dialcache-stale-recovery-policy.test.ts index 40cc38ab..3c8f81db 100644 --- a/typescript/test/dialcache-stale-recovery-policy.test.ts +++ b/typescript/test/dialcache-stale-recovery-policy.test.ts @@ -228,6 +228,14 @@ describe("DialCache stale-recovery error policy", () => { name: "returns a thenable", predicate: vi.fn(() => Promise.resolve(true)) as unknown as StaleRecoveryPredicate, }, + { + name: "returns an object whose then getter throws", + predicate: vi.fn(() => ({ + get then(): never { + throw new Error("then getter failed"); + }, + })) as unknown as StaleRecoveryPredicate, + }, ])("fails closed and preserves the source rejection when the predicate $name", async ({ predicate }) => { const useCase = `DefensiveStaleRecoveryPolicy${predicate.name}`; const sourceError = new Error("source unavailable"); diff --git a/typescript/test/fixtures/formal-witness-attribution.json b/typescript/test/fixtures/formal-witness-attribution.json index 7beea43a..60e37d2d 100644 --- a/typescript/test/fixtures/formal-witness-attribution.json +++ b/typescript/test/fixtures/formal-witness-attribution.json @@ -110,7 +110,7 @@ "model": "formal/models/dialcache-shadow-conformance.qnt", "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/formal-witness-attribution.json/fenced-c1-supersedes-without-repair" }, - "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}, + "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}, "steps": [ {"action":"seed","choice":{"tag":"Some","value":{"#bigint":"3"}},"statePatch":{}}, {"action":"shadowPolicy","choice":{"tag":"Some","value":{"#bigint":"0"}},"statePatch":{}}, diff --git a/typescript/test/fixtures/formal-witness-boundaries.json b/typescript/test/fixtures/formal-witness-boundaries.json index 0cabe278..c22bdb5a 100644 --- a/typescript/test/fixtures/formal-witness-boundaries.json +++ b/typescript/test/fixtures/formal-witness-boundaries.json @@ -128,7 +128,7 @@ "model": "formal/models/dialcache-shadow-conformance.qnt", "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/formal-witness-boundaries.json/same-c1-bytes-confirm-mismatch" }, - "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}, + "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}, "steps": [ {"action":"beginCall","choice":{"tag":"None","value":{"#tup":[]}},"statePatch":{"o":{"calls":[{"#bigint":"0"}],"loaders":{"#bigint":"1"},"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"}}}}, {"action":"rejectLoader","choice":{"tag":"None","value":{"#tup":[]}},"statePatch":{"o":{"calls":[{"#bigint":"3"}]},"d":{"fallbackErrors":["local"]}}}, @@ -174,7 +174,7 @@ "model": "formal/models/dialcache-shadow-conformance.qnt", "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/formal-witness-boundaries.json/different-c1-bytes-supersede" }, - "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}, + "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}, "steps": [ {"action":"beginCall","choice":{"tag":"None","value":{"#tup":[]}},"statePatch":{"o":{"calls":[{"#bigint":"0"}],"loaders":{"#bigint":"1"},"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"}}}}, {"action":"loadFault","choice":{"tag":"Some","value":{"#bigint":"1"}},"statePatch":{}}, @@ -222,7 +222,7 @@ "model": "formal/models/dialcache-shadow-conformance.qnt", "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/formal-witness-boundaries.json/late-shadow-dump-cannot-dispatch-write" }, - "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}, + "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}, "steps": [ {"action":"readFault","choice":{"tag":"Some","value":{"#bigint":"1"}},"statePatch":{}}, {"action":"writeFault","choice":{"tag":"Some","value":{"#bigint":"1"}},"statePatch":{}}, @@ -289,7 +289,7 @@ "model": "formal/models/dialcache-shadow-conformance.qnt", "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/formal-witness-boundaries.json/fill-write-error-preserves-caller" }, - "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}, + "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}, "steps": [ {"action":"advance","choice":{"tag":"Some","value":{"#bigint":"1"}},"statePatch":{}}, {"action":"logPolicy","choice":{"tag":"Some","value":{"#bigint":"1"}},"statePatch":{}}, @@ -312,7 +312,7 @@ "model": "formal/models/dialcache-shadow-conformance.qnt", "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/formal-witness-boundaries.json/dark-read-error-still-allows-source-result" }, - "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}, + "initialState": {"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}, "steps": [ {"action":"readFault","choice":{"tag":"Some","value":{"#bigint":"1"}},"statePatch":{}}, {"action":"beginCall","choice":{"tag":"None","value":{"#tup":[]}},"statePatch":{"o":{"calls":[{"#bigint":"0"}],"loaders":{"#bigint":"1"},"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"}}}}, diff --git a/typescript/test/fixtures/shadow-diagnostics-witnesses.json b/typescript/test/fixtures/shadow-diagnostics-witnesses.json index 1d509b29..d798c866 100644 --- a/typescript/test/fixtures/shadow-diagnostics-witnesses.json +++ b/typescript/test/fixtures/shadow-diagnostics-witnesses.json @@ -7,13 +7,13 @@ "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/omittedLoggingKeepsMismatchMetricsOnlyTest" }, "states": [ - {"input":{"choice":{"#bigint":"2"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}} + {"input":{"choice":{"#bigint":"2"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}} ] } }, @@ -25,14 +25,14 @@ "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/explicitFalseLoggingOverridesEnabledDefaultTest" }, "states": [ - {"input":{"choice":{"#bigint":"6"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"0"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}} + {"input":{"choice":{"#bigint":"6"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}} ] } }, @@ -44,14 +44,14 @@ "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/confirmationPastFreshnessKeepsOriginalPayloadAndAgeTest" }, "states": [ - {"input":{"choice":{"#bigint":"6"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"60000"},"name":"advanceWall"},"mbt::actionTaken":"advanceWall","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"60000"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"60000"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[]}}} + {"input":{"choice":{"#bigint":"6"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"60000"},"name":"advanceWall"},"mbt::actionTaken":"advanceWall","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"60000"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"60000"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"60000"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"1"},"diff":"values","diffNull":false,"entries":[{"cached":{"#bigint":"1"},"path":"","source":{"#bigint":"2"}}],"loggingOutcome":"","paths":[""],"sourceValue":{"#bigint":"2"}}],"loggingOutcomes":["logged"]}}} ] } }, @@ -63,14 +63,14 @@ "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/confirmationRollbackKeepsPayloadAndClampsAgeTest" }, "states": [ - {"input":{"choice":{"#bigint":"6"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"rollbackWall"},"mbt::actionTaken":"rollbackWall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"1000"}}]}}} + {"input":{"choice":{"#bigint":"6"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"rollbackWall"},"mbt::actionTaken":"rollbackWall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"1000"}}],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"1"},"diff":"values","diffNull":false,"entries":[{"cached":{"#bigint":"1"},"path":"","source":{"#bigint":"2"}}],"loggingOutcome":"","paths":[""],"sourceValue":{"#bigint":"2"}}],"loggingOutcomes":["logged"]}}} ] } }, @@ -82,14 +82,14 @@ "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/futureDarkAcquisitionReportsOffsetAndFillsMissTest" }, "states": [ - {"input":{"choice":{"#bigint":"0"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"rollbackWall"},"mbt::actionTaken":"rollbackWall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"1000"}}]}}}, - {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"1000"}}]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseDump"},"mbt::actionTaken":"releaseDump","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[{"#bigint":"60000"}],"writes":{"#bigint":"1"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"1000"}}]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseWrite"},"mbt::actionTaken":"releaseWrite","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["filled"],"sourceScopes":[],"writeTtls":[{"#bigint":"60000"}],"writes":{"#bigint":"1"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"1000"}}]}}} + {"input":{"choice":{"#bigint":"0"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"rollbackWall"},"mbt::actionTaken":"rollbackWall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"1000"}}],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"1000"}}],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseDump"},"mbt::actionTaken":"releaseDump","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[{"#bigint":"60000"}],"writes":{"#bigint":"1"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"1000"}}],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseWrite"},"mbt::actionTaken":"releaseWrite","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"1"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["filled"],"sourceScopes":[],"writeTtls":[{"#bigint":"60000"}],"writes":{"#bigint":"1"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"1000"}}],"mismatchWarnings":[],"loggingOutcomes":[]}}} ] } }, @@ -101,12 +101,331 @@ "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/futureLateDarkReadKeepsTimeoutAndReportsOffsetTest" }, "states": [ - {"input":{"choice":{"#bigint":"0"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"rollbackWall"},"mbt::actionTaken":"rollbackWall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"10"},"name":"advance"},"mbt::actionTaken":"advance","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"10"}}},"s":{"o":{"calls":[{"#bigint":"4"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[]}}}, - {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"4"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"990"}}]}}} + {"input":{"choice":{"#bigint":"0"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"rollbackWall"},"mbt::actionTaken":"rollbackWall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"10"},"name":"advance"},"mbt::actionTaken":"advance","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"10"}}},"s":{"o":{"calls":[{"#bigint":"4"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"4"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":["timeout"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":["local"],"warnings":{"#bigint":"0"},"futureOffsets":[{"layer":"remote_shadow","offsetMs":{"#bigint":"990"}}],"mismatchWarnings":[],"loggingOutcomes":[]}}} + ] + } + }, + { + "regression": "trueShorthandWithValueFalseLogsDiffOnlyTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/trueShorthandWithValueFalseLogsDiffOnlyTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"0"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"6"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"-1"},"count":{"#bigint":"1"},"diff":"values","diffNull":false,"entries":[{"cached":{"#bigint":"1"},"path":"","source":{"#bigint":"2"}}],"loggingOutcome":"","paths":[""],"sourceValue":{"#bigint":"-1"}}],"loggingOutcomes":["logged"]}}} + ] + } + }, + { + "regression": "falseShorthandWithDiffPathsLogsPathsOnlyTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/falseShorthandWithDiffPathsLogsPathsOnlyTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"0"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"7"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"7"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"-1"},"count":{"#bigint":"1"},"diff":"paths","diffNull":false,"entries":[],"loggingOutcome":"","paths":[""],"sourceValue":{"#bigint":"-1"}}],"loggingOutcomes":["logged"]}}} + ] + } + }, + { + "regression": "runtimeFalseShorthandOverridesDefaultDiffTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/runtimeFalseShorthandOverridesDefaultDiffTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"14"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"14"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"0"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}} + ] + } + }, + { + "regression": "runtimeTrueShorthandOverridesDefaultValueTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/runtimeTrueShorthandOverridesDefaultValueTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"15"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"15"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"1"},"diff":"values","diffNull":false,"entries":[{"cached":{"#bigint":"1"},"path":"","source":{"#bigint":"2"}}],"loggingOutcome":"","paths":[""],"sourceValue":{"#bigint":"2"}}],"loggingOutcomes":["logged"]}}} + ] + } + }, + { + "regression": "runtimeOffDiffOverridesDefaultDiffTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/runtimeOffDiffOverridesDefaultDiffTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"4"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"4"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"11"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"11"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"0"},"diff":"off","diffNull":false,"entries":[],"loggingOutcome":"","paths":[],"sourceValue":{"#bigint":"2"}}],"loggingOutcomes":["logged"]}}} + ] + } + }, + { + "regression": "runtimeFalseValueOverridesDefaultValueTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/runtimeFalseValueOverridesDefaultValueTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"4"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"4"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"12"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"12"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"-1"},"count":{"#bigint":"1"},"diff":"values","diffNull":false,"entries":[{"cached":{"#bigint":"1"},"path":"","source":{"#bigint":"2"}}],"loggingOutcome":"","paths":[""],"sourceValue":{"#bigint":"-1"}}],"loggingOutcomes":["logged"]}}} + ] + } + }, + { + "regression": "hookErrorNullsDiffFieldsTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/hookErrorNullsDiffFieldsTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"13"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"13"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"0"},"diff":"values","diffNull":true,"entries":[],"loggingOutcome":"hook_error","paths":[],"sourceValue":{"#bigint":"2"}}],"loggingOutcomes":["hook_error"]}}} + ] + } + }, + { + "regression": "renderErrorNullsUnrenderableSourceTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/renderErrorNullsUnrenderableSourceTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"4"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"4"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"12"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"12"}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"0"},"diff":"values","diffNull":true,"entries":[],"loggingOutcome":"render_error","paths":[],"sourceValue":{"#bigint":"0"}}],"loggingOutcomes":["render_error"]}}} + ] + } + }, + { + "regression": "loggerErrorStillDeliversWarningTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/loggerErrorStillDeliversWarningTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"4"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"4"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"loggerFault"},"mbt::actionTaken":"loggerFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"1"},"diff":"values","diffNull":false,"entries":[{"cached":{"#bigint":"1"},"path":"","source":{"#bigint":"2"}}],"loggingOutcome":"","paths":[""],"sourceValue":{"#bigint":"2"}}],"loggingOutcomes":["logger_error"]}}} + ] + } + }, + { + "regression": "hookErrorOutranksBothFailuresTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/hookErrorOutranksBothFailuresTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"13"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"13"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"loggerFault"},"mbt::actionTaken":"loggerFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"12"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"12"}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"0"},"diff":"values","diffNull":true,"entries":[],"loggingOutcome":"hook_error","paths":[],"sourceValue":{"#bigint":"0"}}],"loggingOutcomes":["hook_error"]}}} + ] + } + }, + { + "regression": "hookErrorOutranksRenderErrorTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/hookErrorOutranksRenderErrorTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"13"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"13"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"12"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"12"}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"0"},"diff":"values","diffNull":true,"entries":[],"loggingOutcome":"hook_error","paths":[],"sourceValue":{"#bigint":"0"}}],"loggingOutcomes":["hook_error"]}}} + ] + } + }, + { + "regression": "renderErrorOutranksLoggerErrorTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/renderErrorOutranksLoggerErrorTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"4"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"4"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"loggerFault"},"mbt::actionTaken":"loggerFault","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"12"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"12"}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"0"},"diff":"values","diffNull":true,"entries":[],"loggingOutcome":"render_error","paths":[],"sourceValue":{"#bigint":"0"}}],"loggingOutcomes":["render_error"]}}} + ] + } + }, + { + "regression": "hookSkippedWithDiffOffTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/hookSkippedWithDiffOffTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"13"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"13"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"5"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"5"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"12"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"12"}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"12"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"0"},"diff":"off","diffNull":false,"entries":[],"loggingOutcome":"render_error","paths":[],"sourceValue":{"#bigint":"0"}}],"loggingOutcomes":["render_error"]}}} + ] + } + }, + { + "regression": "identicalRenderingsLogNoPathsTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/identicalRenderingsLogNoPathsTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"6"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"6"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"1"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"1"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"0"},"diff":"values","diffNull":false,"entries":[],"loggingOutcome":"","paths":[],"sourceValue":{"#bigint":"1"}}],"loggingOutcomes":["logged"]}}} + ] + } + }, + { + "regression": "invalidDiffLeafKeepsValueLeafTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/invalidDiffLeafKeepsValueLeafTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"0"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"8"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"8"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"1"},"count":{"#bigint":"0"},"diff":"off","diffNull":false,"entries":[],"loggingOutcome":"","paths":[],"sourceValue":{"#bigint":"2"}}],"loggingOutcomes":["logged"]}}} + ] + } + }, + { + "regression": "invalidValueLeafKeepsDiffLeafTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/invalidValueLeafKeepsDiffLeafTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"4"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"4"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"9"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"9"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"-1"},"count":{"#bigint":"1"},"diff":"paths","diffNull":false,"entries":[],"loggingOutcome":"","paths":[""],"sourceValue":{"#bigint":"-1"}}],"loggingOutcomes":["logged"]}}} + ] + } + }, + { + "regression": "invalidShorthandKeepsDiffLeafBesideItTest", + "trace": { + "source": { + "model": "formal/models/dialcache-shadow-conformance.qnt", + "recipe": "formal/catalogs/fixture-recipes.json#typescript/test/fixtures/shadow-diagnostics-witnesses.json/invalidShorthandKeepsDiffLeafBesideItTest" + }, + "states": [ + {"input":{"choice":{"#bigint":"0"},"name":"init"},"mbt::actionTaken":"init","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"0"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"10"},"name":"logPolicy"},"mbt::actionTaken":"logPolicy","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"10"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"1"},"name":"seed"},"mbt::actionTaken":"seed","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"1"}}},"s":{"o":{"calls":[],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"0"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"0"},"reads":{"#bigint":"0"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"0"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"beginCall"},"mbt::actionTaken":"beginCall","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"0"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"0"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"2"},"name":"resolveLoader"},"mbt::actionTaken":"resolveLoader","mbt::nondetPicks":{"choice":{"tag":"Some","value":{"#bigint":"2"}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"1"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseLoad"},"mbt::actionTaken":"releaseLoad","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":[],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"0"},"futureOffsets":[],"mismatchWarnings":[],"loggingOutcomes":[]}}}, + {"input":{"choice":{"#bigint":"-1"},"name":"releaseRead"},"mbt::actionTaken":"releaseRead","mbt::nondetPicks":{"choice":{"tag":"None","value":{"#tup":[]}}},"s":{"o":{"calls":[{"#bigint":"2"}],"classifications":{"#bigint":"0"},"comparisons":{"#bigint":"0"},"dumps":{"#bigint":"0"},"invalidations":{"#bigint":"0"},"loaders":{"#bigint":"1"},"loads":{"#bigint":"1"},"maintenance":[],"policyCalls":{"#bigint":"1"},"reads":{"#bigint":"2"},"recovery":[],"shadow":["mismatch"],"sourceScopes":[],"writeTtls":[],"writes":{"#bigint":"0"}},"d":{"ages":[{"#bigint":"0"}],"coalesced":[],"configErrors":{"#bigint":"1"},"fallbackErrors":[],"warnings":{"#bigint":"1"},"futureOffsets":[],"mismatchWarnings":[{"ageMs":{"#bigint":"0"},"cachedValue":{"#bigint":"-1"},"count":{"#bigint":"1"},"diff":"values","diffNull":false,"entries":[{"cached":{"#bigint":"1"},"path":"","source":{"#bigint":"2"}}],"loggingOutcome":"","paths":[""],"sourceValue":{"#bigint":"-1"}}],"loggingOutcomes":["logged"]}}} ] } } diff --git a/typescript/test/formal-protocol-vectors.test.ts b/typescript/test/formal-protocol-vectors.test.ts index 5579debf..7cd67d5c 100644 --- a/typescript/test/formal-protocol-vectors.test.ts +++ b/typescript/test/formal-protocol-vectors.test.ts @@ -19,6 +19,12 @@ import { import { compressPayload, decompressPayload, escapeRawPayload } from "../src/internal/compression.js"; import { deterministicRampSample, deterministicShadowRampSample } from "../src/internal/ramp.js"; +import { + diffShadowMismatchJson, + renderShadowMismatchLog, + type JsonValue, + type ShadowMismatchJsonEntry, +} from "../src/internal/shadow-log-json.js"; interface KeyVector { readonly name: string; @@ -94,6 +100,10 @@ interface ProtocolVectors { readonly rampVectors: ReadonlyArray<{ name: string; input: KeyVector["input"]; layer: "local" | "remote" | "shadow"; sample: number; }>; + // An omitted cached or source member is an absent side (D13), TypeScript undefined. + readonly mismatchPathVectors: ReadonlyArray<{ + name: string; cached?: JsonValue; source?: JsonValue; entries: readonly ShadowMismatchJsonEntry[]; count: number; + }>; } const corpusUrl = new URL("../../formal/tools/vector-artifacts.mjs", import.meta.url).href; @@ -253,6 +263,27 @@ describe("formal protocol conformance vectors", () => { }); } + // W10: the built-in walk over two JSON values, its entries compared as + // parsed JSON. Both projections of one entry list are checked: the paths + // and count at diff "paths", and the entries with their values at "values". + for (const vector of vectors.mismatchPathVectors) { + it(`computes mismatch paths: ${vector.name}`, () => { + const cached = Object.hasOwn(vector, "cached") ? vector.cached : undefined; + const source = Object.hasOwn(vector, "source") ? vector.source : undefined; + if (cached !== undefined && source !== undefined) { + expect(diffShadowMismatchJson(cached, source)).toStrictEqual({ entries: vector.entries, count: vector.count }); + } + const paths = vector.entries.map((entry) => entry.path); + const values = renderShadowMismatchLog({ value: false, diff: "values" }, cached, source); + expect(values.failure).toBeNull(); + expect(values.fields.mismatchPaths).toStrictEqual(paths); + expect(values.fields.mismatchPathCount).toBe(vector.count); + expect(JSON.parse(values.fields.diffJson!)).toStrictEqual(vector.entries); + expect(renderShadowMismatchLog({ value: false, diff: "paths" }, cached, source)) + .toStrictEqual({ fields: { mismatchPaths: paths, mismatchPathCount: vector.count }, failure: null }); + }); + } + for (const vector of vectors.rampVectors) { it(`assigns deterministic cohort: ${vector.name}`, () => { const key = new DialCacheKey(vector.input); diff --git a/typescript/test/formal-rust-replay.test.ts b/typescript/test/formal-rust-replay.test.ts index c0f85d8d..bf90dbc7 100644 --- a/typescript/test/formal-rust-replay.test.ts +++ b/typescript/test/formal-rust-replay.test.ts @@ -134,9 +134,12 @@ describe("Rust replay report gate", () => { describe("Rust validation lanes", () => { const directory = "/checkout"; it("checks formatting, clippy and default tests from inside the crate so rustup honors its toolchain pin", () => { + // The second clippy run is the only Rust step that builds the library + // without test-util, which replaces shipped code when it is on. expect(validationPlan("check-rust", { directory }).map(step => [step.command, ...step.args!])).toEqual([ ["cargo", "fmt", "--check"], ["cargo", "clippy", "--all-targets", "--all-features", "--", "-D", "warnings"], + ["cargo", "clippy", "--lib", "--features", "redis,prometheus", "--", "-D", "warnings"], ["cargo", "test", "--all-features"], ]); for (const step of validationPlan("check-rust", { directory })) { diff --git a/typescript/test/formal-rust-semantic-runner.test.ts b/typescript/test/formal-rust-semantic-runner.test.ts index ab7d02fb..8913ff2c 100644 --- a/typescript/test/formal-rust-semantic-runner.test.ts +++ b/typescript/test/formal-rust-semantic-runner.test.ts @@ -148,8 +148,17 @@ describe("Rust fault catalog", () => { expect(entry.case, entry.id).toBe(counterpart.case); expect(entry.description, entry.id).toBe(counterpart.description); expect(entry.typescriptRequiredDetections, entry.id).toEqual(counterpart.typescript.requiredDetections); - expect(entry.requiredDetections, entry.id).toEqual(expect.arrayContaining(["generated", "portable"])); + // A rule only the checked-in vectors reach, such as the built-in + // mismatch walk over nested values, requires fixed in place of + // generated, as the shared catalog allows. + expect(entry.requiredDetections, entry.id).toContain("portable"); + expect(entry.requiredDetections.some(cohort => ["generated", "fixed"].includes(cohort)), entry.id).toBe(true); expect(entry.requiredDetections.every(cohort => ["generated", "fixed", "portable"].includes(cohort)), entry.id).toBe(true); + // Only those vector-only rules may trade generated for fixed: the twin of + // a mutant the generated histories must detect keeps that requirement. + if (counterpart.typescript.requiredDetections.includes("generated")) { + expect(entry.requiredDetections, entry.id).toContain("generated"); + } } }); diff --git a/typescript/test/formal-shadow-warning-witnesses.test.ts b/typescript/test/formal-shadow-warning-witnesses.test.ts new file mode 100644 index 00000000..9c49b860 --- /dev/null +++ b/typescript/test/formal-shadow-warning-witnesses.test.ts @@ -0,0 +1,380 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterAll, describe, expect, it } from "vitest"; + +import { parseTrace, profiles, projectObservation } from "../../formal/replay/features.mjs"; +import { shadowWitnesses } from "../../formal/replay/witnesses/shadow.mjs"; +import { witnessStates } from "../../formal/replay/witnesses/trace.mjs"; +import { emptyObservation, type Observation, type ObservedEvent } from "./formal/behavior-driver.js"; +import { replayThroughCoordinator } from "./formal/coordinated-replay.js"; + +// Negative controls for the shadow profile's warning content (record D12): +// the d channel parser, the projection of a driver's mismatchWarning and +// shadowMismatchLogging events, the classifier labels that credit a warning, +// and the TypeScript replay of the Quint regressions that pin them. The +// histories are the committed Quint excerpts of the regressions; the +// corruptions below receive no conformance or mutation credit. +type State = { input: { name: string; choice: unknown }; s: { o: Record; d: Record } }; +type Fixture = { regression: string; trace: { states: State[] } }; +const fixtures = JSON.parse(readFileSync(new URL("./fixtures/shadow-diagnostics-witnesses.json", import.meta.url), "utf8")) as Fixture[]; +const integer = (value: number) => ({ "#bigint": String(value) }); +const shadow = profiles.shadow!; +// A history's states with the optional simulator annotations dropped, so an +// edited explicit input stays the one authoritative command (PORTING.md). +function statesOf(regression: string): State[] { + const fixture = fixtures.find(item => item.regression === regression); + if (fixture === undefined) throw new Error(`Missing fixture ${regression}`); + return structuredClone(fixture.trace.states).map(state => + Object.fromEntries(Object.entries(state).filter(([key]) => !key.startsWith("mbt::"))) as State); +} +function labels(states: State[]): Set { + const raw = { states }, path = "trace.itf.json"; + return shadowWitnesses([{ ...parseTrace(raw, path, shadow), ...witnessStates(raw, path) }]); +} +// The final step's d channel, edited in place. +function finalDiagnostics(states: State[]): Record { return states.at(-1)!.s.d; } +type Warning = Record; +function finalWarning(states: State[]): Warning { + return (finalDiagnostics(states).mismatchWarnings as Warning[])[0]!; +} +// hookErrorOutranksRenderErrorTest with `value` written false at runtime +// (logPolicy 12) over the default `logMismatches: true`: the warning carries +// only the null diff fields, so nothing but the hook needed the source's +// rendering and the history cannot show hook_error outranking render_error. +function valueOffPrecedenceStates(): State[] { + const states = statesOf("hookErrorOutranksRenderErrorTest"); + const overlay = structuredClone(states[0]!); + overlay.input = { name: "logPolicy", choice: integer(12) }; + states.splice(1, 0, overlay); + Object.assign(finalWarning(states), { cachedValue: integer(-1), sourceValue: integer(-1) }); + return states; +} + +describe("shadow warning witness labels", () => { + it("credits each failure outcome from the public outcome, not the inputs alone", () => { + const states = statesOf("hookErrorNullsDiffFieldsTest"); + expect(labels(states)).toContain("logging-outcome:hook_error"); + finalDiagnostics(states).loggingOutcomes = ["logged"]; + const changed = labels(states); + expect(changed).not.toContain("logging-outcome:hook_error"); + expect(changed).toContain("logging-outcome:logged"); + }); + + it("requires every failure input for a precedence witness", () => { + const states = statesOf("hookErrorOutranksBothFailuresTest"); + const original = labels(states); + expect(original).toContain("hook-error-over-render-error"); + expect(original).toContain("hook-error-over-logger-error"); + // The same public history with the logger's fault turned off at its input. + const loggerFault = states.findIndex(state => state.input.name === "loggerFault"); + states[loggerFault]!.input.choice = integer(0); + const withoutLoggerFault = labels(states); + expect(withoutLoggerFault).toContain("hook-error-over-render-error"); + expect(withoutLoggerFault).not.toContain("hook-error-over-logger-error"); + }); + + it("requires the null source field for an unrenderable source", () => { + const states = statesOf("renderErrorNullsUnrenderableSourceTest"); + expect(labels(states)).toContain("unrenderable-source-logs-null"); + finalWarning(states).sourceValue = integer(2); + expect(labels(states)).not.toContain("unrenderable-source-logs-null"); + }); + + it("requires the field groups a shorthand row captures", () => { + const row1 = statesOf("trueShorthandWithValueFalseLogsDiffOnlyTest"); + expect(labels(row1)).toContain("shorthand-row:1"); + finalWarning(row1).cachedValue = integer(1); + finalWarning(row1).sourceValue = integer(2); + expect(labels(row1)).not.toContain("shorthand-row:1"); + const row3 = statesOf("runtimeFalseShorthandOverridesDefaultDiffTest"); + expect(labels(row3)).toContain("shorthand-row:3"); + finalDiagnostics(row3).mismatchWarnings = [finalWarning(statesOf("runtimeTrueShorthandOverridesDefaultValueTest"))]; + finalDiagnostics(row3).loggingOutcomes = ["logged"]; + finalDiagnostics(row3).warnings = integer(1); + expect(labels(row3)).not.toContain("shorthand-row:3"); + }); + + it("requires a leaf written off over an inherited enabled leaf, and the other leaf kept", () => { + const diffOff = statesOf("runtimeOffDiffOverridesDefaultDiffTest"); + expect(labels(diffOff)).toContain("written-off-leaf:diff"); + // The diff fields the inherited `values` leaf would have logged. + finalDiagnostics(diffOff).mismatchWarnings = [finalWarning(statesOf("runtimeTrueShorthandOverridesDefaultValueTest"))]; + expect(labels(diffOff)).not.toContain("written-off-leaf:diff"); + // The same warning under fixture 0, whose defaults inherit nothing. + const uninherited = statesOf("runtimeOffDiffOverridesDefaultDiffTest"); + uninherited[0]!.input.choice = integer(0); + expect(labels(uninherited)).not.toContain("written-off-leaf:diff"); + const valueFalse = statesOf("runtimeFalseValueOverridesDefaultValueTest"); + expect(labels(valueFalse)).toContain("written-off-leaf:value"); + finalWarning(valueFalse).cachedValue = integer(1); + finalWarning(valueFalse).sourceValue = integer(2); + expect(labels(valueFalse)).not.toContain("written-off-leaf:value"); + const unwrittenOverlay = statesOf("runtimeFalseValueOverridesDefaultValueTest"); + unwrittenOverlay.find(state => state.input.name === "logPolicy")!.input.choice = integer(4); + expect(labels(unwrittenOverlay)).not.toContain("written-off-leaf:value"); + }); + + it("requires the logged content beside a logger failure", () => { + const states = statesOf("loggerErrorStillDeliversWarningTest"); + expect(labels(states)).toContain("logger-error-keeps-warning-content"); + finalWarning(states).diffNull = true; + expect(labels(states)).not.toContain("logger-error-keeps-warning-content"); + }); + + it("requires logged value fields with a null source for hook_error over render_error", () => { + const states = statesOf("hookErrorOutranksRenderErrorTest"); + expect(labels(states)).toContain("hook-error-over-render-error"); + const valueOff = valueOffPrecedenceStates(); + expect(labels(valueOff)).toContain("logging-outcome:hook_error"); + expect(labels(valueOff)).not.toContain("hook-error-over-render-error"); + finalWarning(states).sourceValue = integer(2); + expect(labels(states)).not.toContain("hook-error-over-render-error"); + }); + + it("requires a public render_error, a failing logger and no diff hook for render_error over logger_error", () => { + const states = statesOf("renderErrorOutranksLoggerErrorTest"); + expect(labels(states)).toContain("render-error-over-logger-error"); + finalDiagnostics(states).loggingOutcomes = ["logger_error"]; + expect(labels(states)).not.toContain("render-error-over-logger-error"); + const loggerOff = statesOf("renderErrorOutranksLoggerErrorTest"); + loggerOff.find(state => state.input.name === "loggerFault")!.input.choice = integer(0); + expect(labels(loggerOff)).not.toContain("render-error-over-logger-error"); + // Fixture 13 is fixture 4 with the throwing diff hook, whose failure would outrank. + const hooked = statesOf("renderErrorOutranksLoggerErrorTest"); + hooked[0]!.input.choice = integer(13); + expect(labels(hooked)).not.toContain("render-error-over-logger-error"); + }); + + it("requires a present diff hook and an outcome it did not name for a hook skipped at diff off", () => { + const states = statesOf("hookSkippedWithDiffOffTest"); + expect(labels(states)).toContain("hook-skipped-at-diff-off"); + finalDiagnostics(states).loggingOutcomes = ["hook_error"]; + finalWarning(states).loggingOutcome = "hook_error"; + expect(labels(states)).not.toContain("hook-skipped-at-diff-off"); + // Fixture 4 is fixture 13 without the hook: nothing was there to skip. + const noHook = statesOf("hookSkippedWithDiffOffTest"); + noHook[0]!.input.choice = integer(4); + expect(labels(noHook)).not.toContain("hook-skipped-at-diff-off"); + }); + + it("requires listed diff fields with no path for identical renderings and one root path for differing ones", () => { + expect(labels(statesOf("identicalRenderingsLogNoPathsTest"))).toContain("identical-renderings-log-no-paths"); + const rootEntry = { count: integer(1), paths: [""], entries: [{ path: "", cached: integer(1), source: integer(1) }] }; + for (const corrupt of [rootEntry, { count: integer(1) }, { diffNull: true }]) { + const changed = statesOf("identicalRenderingsLogNoPathsTest"); + Object.assign(finalWarning(changed), corrupt); + expect(labels(changed), JSON.stringify(corrupt)).not.toContain("identical-renderings-log-no-paths"); + } + const differing = statesOf("falseShorthandWithDiffPathsLogsPathsOnlyTest"); + expect(labels(differing)).toContain("differing-renderings-log-root-entry"); + for (const corrupt of [{ paths: ["/n"] }, { count: integer(2) }, { diffNull: true }]) { + const changed = statesOf("falseShorthandWithDiffPathsLogsPathsOnlyTest"); + Object.assign(finalWarning(changed), corrupt); + expect(labels(changed), JSON.stringify(corrupt)).not.toContain("differing-renderings-log-root-entry"); + } + }); + + it("credits the field groups the warning carries, not the leaves the inputs capture", () => { + const states = statesOf("loggerErrorStillDeliversWarningTest"); + expect(labels(states)).toContain("warning-fields:value+values"); + const withFields = (fields: Warning) => { + const changed = statesOf("loggerErrorStillDeliversWarningTest"); + Object.assign(finalWarning(changed), fields); + return labels(changed); + }; + const omitted = { cachedValue: integer(-1), sourceValue: integer(-1) }; + const noDiff = { diff: "off", count: integer(0), paths: [], entries: [] }; + for (const [fields, label] of [[omitted, "warning-fields:values"], [noDiff, "warning-fields:value"], + [{ diff: "paths", entries: [] }, "warning-fields:value+paths"]] as const) { + const changed = withFields(fields); + expect(changed, label).toContain(label); + expect(changed, label).not.toContain("warning-fields:value+values"); + } + }); + + it("requires the shorthand rows' inputs and public fields", () => { + const row2 = statesOf("falseShorthandWithDiffPathsLogsPathsOnlyTest"); + expect(labels(row2)).toContain("shorthand-row:2"); + Object.assign(finalWarning(row2), { cachedValue: integer(1), sourceValue: integer(2) }); + expect(labels(row2)).not.toContain("shorthand-row:2"); + const row2Values = statesOf("falseShorthandWithDiffPathsLogsPathsOnlyTest"); + Object.assign(finalWarning(row2Values), { diff: "values", entries: [{ path: "", cached: integer(1), source: integer(2) }] }); + expect(labels(row2Values)).not.toContain("shorthand-row:2"); + // logPolicy 3 writes diff paths without the false shorthand: the same warning. + const row2Unwritten = statesOf("falseShorthandWithDiffPathsLogsPathsOnlyTest"); + row2Unwritten.find(state => state.input.name === "logPolicy")!.input.choice = integer(3); + expect(labels(row2Unwritten)).not.toContain("shorthand-row:2"); + const row4 = statesOf("runtimeTrueShorthandOverridesDefaultValueTest"); + expect(labels(row4)).toContain("shorthand-row:4"); + Object.assign(finalWarning(row4), { cachedValue: integer(-1), sourceValue: integer(-1) }); + expect(labels(row4)).not.toContain("shorthand-row:4"); + // Fixture 0 is fixture 15 without the value false default the shorthand replaces. + const row4Uninherited = statesOf("runtimeTrueShorthandOverridesDefaultValueTest"); + row4Uninherited[0]!.input.choice = integer(0); + expect(labels(row4Uninherited)).not.toContain("shorthand-row:4"); + }); + + it("requires the reported configuration error, the invalid input and the one valid leaf it keeps", () => { + const unreported = (states: State[]) => { + const admission = states.findIndex(state => state.input.name === "beginCall"); + for (const state of states.slice(admission)) state.s.d.configErrors = integer(0); + return states; + }; + for (const regression of ["invalidDiffLeafKeepsValueLeafTest", "invalidValueLeafKeepsDiffLeafTest"]) { + expect(labels(statesOf(regression)), regression).toContain("invalid-leaf-keeps-valid-leaf"); + expect(labels(unreported(statesOf(regression))), regression).not.toContain("invalid-leaf-keeps-valid-leaf"); + const both = statesOf(regression); + finalDiagnostics(both).mismatchWarnings = [finalWarning(statesOf("runtimeTrueShorthandOverridesDefaultValueTest"))]; + expect(labels(both), regression).not.toContain("invalid-leaf-keeps-valid-leaf"); + } + // logPolicy 5 writes value true and the false shorthand, valid inputs that log the same value fields. + const valid = statesOf("invalidDiffLeafKeepsValueLeafTest"); + valid.find(state => state.input.name === "logPolicy")!.input.choice = integer(5); + expect(labels(valid)).not.toContain("invalid-leaf-keeps-valid-leaf"); + const shorthand = statesOf("invalidShorthandKeepsDiffLeafBesideItTest"); + expect(labels(shorthand)).toContain("invalid-shorthand-keeps-written-leaf"); + expect(labels(unreported(statesOf("invalidShorthandKeepsDiffLeafBesideItTest")))).not.toContain("invalid-shorthand-keeps-written-leaf"); + for (const corrupt of [{ cachedValue: integer(1), sourceValue: integer(2) }, { diff: "paths", entries: [] }]) { + const changed = statesOf("invalidShorthandKeepsDiffLeafBesideItTest"); + Object.assign(finalWarning(changed), corrupt); + expect(labels(changed), JSON.stringify(corrupt)).not.toContain("invalid-shorthand-keeps-written-leaf"); + } + // logPolicy 4 writes diff values without the malformed shorthand: the same warning. + const wellFormed = statesOf("invalidShorthandKeepsDiffLeafBesideItTest"); + wellFormed.find(state => state.input.name === "logPolicy")!.input.choice = integer(4); + expect(labels(wellFormed)).not.toContain("invalid-shorthand-keeps-written-leaf"); + }); +}); + +describe("shadow d channel parser", () => { + it("requires both warning lists for the profile that asserts them", () => { + const states = statesOf("hookErrorNullsDiffFieldsTest"); + expect(parseTrace({ states }, "trace", shadow).steps.at(-1)!.diagnostics!.loggingOutcomes).toEqual(["hook_error"]); + delete finalDiagnostics(states).mismatchWarnings; + expect(() => parseTrace({ states }, "trace", shadow)).toThrow(/invalid diagnostics/); + }); + + it("rejects a warning record outside the shared shape", () => { + const states = statesOf("hookErrorNullsDiffFieldsTest"); + finalWarning(states).cacheKey = "urn:id:1#Behavior"; + expect(() => parseTrace({ states }, "trace", shadow)).toThrow(/invalid mismatch warning/); + const unknownOutcome = statesOf("hookErrorNullsDiffFieldsTest"); + finalWarning(unknownOutcome).loggingOutcome = "logger_error"; + expect(() => parseTrace({ states: unknownOutcome }, "trace", shadow)).toThrow(/invalid mismatch warning/); + }); + + it("accepts the lists absent or empty, and only so, for a profile that does not assert them", () => { + const raw = JSON.parse(readFileSync(new URL("../../formal/generated/dark-layers-smoke.itf.json", import.meta.url), "utf8")) as { states: State[] }; + const darkLayers = profiles["dark-layers"]!; + expect(raw.states[0]!.s.d).not.toHaveProperty("mismatchWarnings"); + const absent = parseTrace(structuredClone(raw), "absent", darkLayers); + for (const state of raw.states) Object.assign(state.s.d, { mismatchWarnings: [], loggingOutcomes: [] }); + const empty = parseTrace(structuredClone(raw), "empty", darkLayers); + expect(empty.steps.map(step => step.diagnostics)).toEqual(absent.steps.map(step => step.diagnostics)); + expect(empty.steps[0]!.diagnostics).not.toHaveProperty("mismatchWarnings"); + raw.states[1]!.s.d.loggingOutcomes = ["logged"]; + expect(() => parseTrace(raw, "carried", darkLayers)).toThrow(/invalid diagnostics/); + }); +}); + +describe("shadow warning projection", () => { + const labels = { cacheNamespace: "urn", useCase: "Behavior", keyType: "id" }; + const warning = (fields: Record): ObservedEvent => ({ event: "mismatchWarning", ...labels, outcome: "mismatch", + cacheKey: "urn:id:1#Behavior", cachedValueAgeSeconds: 0, ...fields } as ObservedEvent); + const observed = (event: ObservedEvent, outcome = "logged"): Observation => ({ + ...emptyObservation({ policy: {}, observe: [] }), calls: [{ status: "value", value: 2 }], shadow: ["mismatch"], + events: [{ event: "shadowAge", ...labels, outcome: "mismatch", seconds: 0 }, event, + { event: "shadowMismatchLogging", ...labels, outcome } as ObservedEvent], + }); + const projected = (event: ObservedEvent, outcome?: string) => + (projectObservation(shadow, observed(event, outcome)).d as { mismatchWarnings: unknown[]; loggingOutcomes: string[] }); + const everything = { mismatchPathsJson: "[\"\"]", mismatchPathCount: 1, diffJson: "[{\"path\":\"\",\"cached\":1,\"source\":2}]", + cachedValueJson: "1", sourceValueJson: "2" }; + + it("projects the warning's fields as parsed JSON and its one outcome", () => { + expect(projected(warning(everything))).toEqual({ ...projected(warning(everything)), loggingOutcomes: ["logged"], mismatchWarnings: [{ + ageSeconds: 0, cachedValue: 1, sourceValue: 2, diff: "values", diffNull: false, count: 1, paths: [""], + entries: [{ path: "", cached: 1, source: 2 }], loggingOutcome: "", + }] }); + // Rendering text is a native detail: a port's spelling of the same JSON projects alike. + expect(projected(warning({ ...everything, diffJson: "[ {\"source\": 2, \"path\": \"\", \"cached\": 1} ]" })).mismatchWarnings) + .toEqual(projected(warning(everything)).mismatchWarnings); + expect(projected(warning({ mismatchPathsJson: null, mismatchPathCount: null, diffJson: null, loggingOutcome: "hook_error" }), "hook_error")) + .toMatchObject({ loggingOutcomes: ["hook_error"], mismatchWarnings: [{ diff: "values", diffNull: true, cachedValue: -1, paths: [], entries: [] }] }); + }); + + it("refuses diff fields that are not carried together or not null together", () => { + expect(() => projected(warning({ ...everything, diffJson: null }))).toThrow(/observed value domain/); + expect(() => projected(warning({ diffJson: "[]" }))).toThrow(/observed value domain/); + expect(() => projected(warning({ mismatchPathsJson: "[]" }))).toThrow(/observed value domain/); + expect(() => projected(warning({ ...everything, mismatchPathsJson: "not json" }))).toThrow(/observed value domain/); + }); + + it("refuses a value outside the profile's domain and a warning without its age", () => { + expect(() => projected(warning({ cachedValueJson: "{\"a\":1}", sourceValueJson: "2" }))).toThrow(/observed value domain/); + const { cachedValueAgeSeconds: _age, ...withoutAge } = warning(everything); + expect(() => projected(withoutAge as ObservedEvent)).toThrow(/Missing actual warning age/); + }); + + it("keeps a missing entry side apart from a null one", () => { + const sides = (diffJson: string) => (projected(warning({ ...everything, diffJson })).mismatchWarnings[0] as { entries: unknown[] }).entries; + expect(sides("[{\"path\":\"\",\"source\":2}]")).toEqual([{ path: "", cached: -1, source: 2 }]); + expect(sides("[{\"path\":\"\",\"cached\":null,\"source\":2}]")).toEqual([{ path: "", cached: 6, source: 2 }]); + }); +}); + +describe("shadow warning regressions replayed in TypeScript", () => { + const directory = mkdtempSync(join(tmpdir(), "dialcache-shadow-warning-")); + afterAll(() => rmSync(directory, { recursive: true, force: true })); + const write = (name: string, states: State[]) => { + const path = join(directory, `${name}.itf.json`); + writeFileSync(path, JSON.stringify({ states })); + return path; + }; + + for (const fixture of fixtures) it(`replays ${fixture.regression}`, async () => { + await expect(replayThroughCoordinator("shadow", write(fixture.regression, structuredClone(fixture.trace.states)))).resolves.toMatchObject({}); + }); + + it("detects a logger that did not fail where the history says it did", async () => { + const states = statesOf("loggerErrorStillDeliversWarningTest"); + states.find(state => state.input.name === "loggerFault")!.input.choice = integer(0); + await expect(replayThroughCoordinator("shadow", write("loggerOff", states))).rejects.toThrow(/expected[\s\S]*actual/); + }); + + it("detects a source that rendered where the history says it had no rendering", async () => { + // The caller's result follows the changed input, so only the warning and + // its outcome still claim the unrenderable source. + const states = statesOf("renderErrorNullsUnrenderableSourceTest"); + const resolved = states.findIndex(state => state.input.name === "resolveLoader"); + states[resolved]!.input.choice = integer(2); + for (const state of states.slice(resolved)) state.s.o.calls = [integer(2)]; + await expect(replayThroughCoordinator("shadow", write("rendered", states))).rejects.toThrow(/expected[\s\S]*actual/); + }); + + const writtenOff = [["runtimeOffDiffOverridesDefaultDiffTest", 11], ["runtimeFalseValueOverridesDefaultValueTest", 12]] as const; + for (const [regression, choice] of writtenOff) { + it(`detects an inherited leaf that won over logPolicy ${choice}`, async () => { + // logPolicy 1 writes nothing off over the same default: an implementation + // that read the written-off leaf as unwritten logs what it logs. + const states = statesOf(regression); + const overlay = states.find(state => state.input.name === "logPolicy")!; + expect(overlay.input.choice).toEqual(integer(choice)); + overlay.input.choice = integer(1); + await expect(replayThroughCoordinator("shadow", write(`inherited${choice}`, states))).rejects.toThrow(/expected[\s\S]*actual/); + }); + } + + it("detects a diff hook that did not run where the history says it threw", async () => { + // Fixture 4 is fixture 13 without the hook: the same calls and verdicts. + const states = statesOf("hookErrorNullsDiffFieldsTest"); + states[0]!.input.choice = integer(4); + await expect(replayThroughCoordinator("shadow", write("noHook", states))).rejects.toThrow(/expected[\s\S]*actual/); + }); + + it("replays the value-off history the hook_error over render_error label refuses", async () => { + await expect(replayThroughCoordinator("shadow", write("valueOffPrecedence", valueOffPrecedenceStates()))).resolves.toMatchObject({}); + }); +}); diff --git a/typescript/test/formal-validation.test.ts b/typescript/test/formal-validation.test.ts index 9658016a..50f4b473 100644 --- a/typescript/test/formal-validation.test.ts +++ b/typescript/test/formal-validation.test.ts @@ -504,24 +504,48 @@ describe("full formal workflow shape", () => { for (const lane of lanes) expect(needsOf(jobs[lane]!), lane).toEqual(["generate"]); }); + it("gives the model check room for every catalogued model fault on a hosted runner", async () => { + // make formal-check runs every scheduled model, then measures every catalogued model fault on + // the Quint pool, then checks the lint baseline and the kernel fixtures; setup and the evidence + // upload take under a minute. Run 39239127644 on main spent 6.0 minutes on the models, 528 + // worker-minutes on its 79 challenges (6.7 each) and 2.3 minutes on the rest. Run 40245022083 + // spent 6.2 minutes on the models and finished 78 of 90 challenges at 7.1 worker-minutes each, + // the longest 13, before the former 150-minute timeout. The pool finishes the campaign within + // its worker-minutes over the workers plus one challenge's straggling share (list scheduling), + // so growing the catalog fails here until the timeout grows. + const { resolveConcurrency } = await import(new URL("../../formal/tools/quint-pool.mjs", import.meta.url).href) as { + resolveConcurrency(env: Record, available?: number, memory?: number): number; + }; + const challenges = (JSON.parse(readFileSync(new URL("../../formal/catalogs/execution.json", import.meta.url), "utf8")) as { challenges: unknown[] }).challenges.length; + // A hosted ubuntu-latest runner: four cores and 16 GB, at 2 GiB a worker. + const workers = resolveConcurrency({}, 4, 16e9); + expect(workers).toBe(4); + const setupMinutes = 1, modelMinutes = 6.5, workerMinutesPerChallenge = 7.1, longestChallengeMinutes = 13, closingMinutes = 2.5; + const campaign = challenges * workerMinutesPerChallenge / workers + longestChallengeMinutes * (1 - 1 / workers); + expect(setupMinutes + modelMinutes + campaign + closingMinutes, `${challenges} challenges`) + .toBeLessThanOrEqual(jobs["check-models"]!["timeout-minutes"]!); + }); + it("shards both mutation lanes so every shard fits its timeout on the slow runner class, and gates the aggregate on their merges", () => { const uploadOf = (job: Job) => job.steps.find(step => step.uses?.startsWith("actions/upload-artifact"))!; const downloadsOf = (job: Job) => job.steps.filter(step => step.uses?.startsWith("actions/download-artifact")).map(step => step.with); const matrixShard = "$" + "{{ matrix.shard }}"; - // A shard runs the baselines, then its slice of the catalog strictly in sequence; on the slow - // runner class a TypeScript mutant costs about 2 minutes and a Go mutant about 3.5 (run - // 35335831285 measured 200-208 s per Go mutant on that class), and one hung - // cohort adds its own bound (the 540 s vitest spawn timeout, the 480 s go test timeout) before - // the shard fails. The matrix must keep every shard inside the job timeout, so growing the - // catalog fails here until the matrix grows. The shard count in the matrix and in MUTATION_SHARD - // must agree or the merge refuses the shards. + // A shard sets up, runs the baselines, then its slice of the catalog strictly in sequence; on + // the slow runner class a TypeScript mutant costs about 2 minutes and a Go mutant about 4.5 + // (run 40146831443 measured about 270 s per Go mutant on that class), and one hung cohort + // adds its own bound (the 540 s vitest spawn timeout, the 480 s go test timeout) before the + // shard fails. From job start to the last baseline, the 53 shards of runs 40142189738, + // 40146831443 and 40245022083 took 1.0 to 1.6 minutes in TypeScript and 2.7 to 5.3 in Go, + // which compiles and runs every generated leaf test, so each lane carries its own baseline. The + // matrix must keep every shard inside the job timeout, so growing the catalog fails here until + // the matrix grows. The shard count in the matrix and in MUTATION_SHARD must agree or the merge + // refuses the shards. const catalogSize = (JSON.parse(readFileSync(new URL("../../formal/catalogs/mutations.json", import.meta.url), "utf8")) as { mutations: unknown[] }).mutations.length; - const baselineMinutes = 4; const table = [ - { lane: "typescript-mutations", language: "ts", output: ".formal-traces/semantic", artifact: "typescript-semantic", timeout: 40, shards: 6, slowMinutesPerMutant: 2, hungCohortMinutes: 9, go: undefined }, - { lane: "go-mutations", language: "go", output: ".formal-traces/go-semantic", artifact: "go-semantic", timeout: 40, shards: 10, slowMinutesPerMutant: 3.5, hungCohortMinutes: 8, go: { go: "true" } }, + { lane: "typescript-mutations", language: "ts", output: ".formal-traces/semantic", artifact: "typescript-semantic", timeout: 40, shards: 6, baselineMinutes: 2, slowMinutesPerMutant: 2, hungCohortMinutes: 9, go: undefined }, + { lane: "go-mutations", language: "go", output: ".formal-traces/go-semantic", artifact: "go-semantic", timeout: 40, shards: 15, baselineMinutes: 5.5, slowMinutesPerMutant: 4.5, hungCohortMinutes: 8, go: { go: "true" } }, ]; - for (const { lane, language, output, artifact, timeout, shards, slowMinutesPerMutant, hungCohortMinutes, go } of table) { + for (const { lane, language, output, artifact, timeout, shards, baselineMinutes, slowMinutesPerMutant, hungCohortMinutes, go } of table) { const job = jobs[lane]!; const perShard = Math.ceil(catalogSize / shards); expect(baselineMinutes + perShard * slowMinutesPerMutant + hungCohortMinutes, `${lane}: ${perShard} mutants per shard`).toBeLessThanOrEqual(timeout); diff --git a/typescript/test/formal/behavior-driver.ts b/typescript/test/formal/behavior-driver.ts index 4b9cd2c7..d7585a80 100644 --- a/typescript/test/formal/behavior-driver.ts +++ b/typescript/test/formal/behavior-driver.ts @@ -23,7 +23,7 @@ export type Recovery = "allow" | "deny" | "error"; export type EventName = "readContext" | "readAbort" | "request" | "miss" | "disabled" | "error" | "coalesced" | "invalidation" | "shadowAge" | "recoveryAge" | "futureOffset" | "size" | "storedSize" | "compression" | "get" | "fallback" | "serialization" - | "mismatchWarning" | "writeDispatch" | "marker" | "coalescingState"; + | "mismatchWarning" | "shadowMismatchLogging" | "writeDispatch" | "marker" | "coalescingState"; export interface ObservedEvent { event: EventName; [field: string]: string | number | boolean | null } // JSON-shaped adapter observations deliberately include malformed replies. A // strongly typed port can reject these at its adapter boundary instead. @@ -38,6 +38,7 @@ export interface Fixture { shadowMaxInFlight?: number; recovery?: Recovery | "default"; comparator?: "equal" | "unequal" | "error"; + mismatchDiff?: "error"; comparisonMs?: number; sourceWorkMs?: number; shadowHook?: boolean; @@ -48,12 +49,12 @@ export interface Fixture { } export interface Faults { localStorage: boolean; - read: boolean; write: boolean; dump: boolean; load: boolean; policy: boolean; observer: boolean; + read: boolean; write: boolean; dump: boolean; load: boolean; policy: boolean; observer: boolean; logger: boolean; holdReads: boolean; holdWrites: boolean; holdDumps: boolean; holdLoads: boolean; holdPolicies: boolean; } export type Input = | { op: "begin"; key?: string; useCase?: string; instance?: string; scope?: string; outside?: boolean; disabled?: boolean; recovery?: Recovery } - | { op: "resolve"; loader: number; value?: Value } + | { op: "resolve"; loader: number; value?: Value; unrenderable?: true } | { op: "reject"; loader: number; error?: "timeout" } | { op: "advance"; ms: number; deliverTimers?: boolean } | { op: "shiftWall"; ms: number } @@ -68,7 +69,7 @@ export type Input = | { op: "openScope"; id: string; parent?: string; disabled?: boolean; instance?: string } | { op: "closeScope"; id: string }; -export type CallResult = { status: "pending" } | { status: "value"; value: Value | { absent: true } } +export type CallResult = { status: "pending" } | { status: "value"; value: Value | { absent: true } | { unrenderable: true } } | { status: "error"; error: string }; export interface Observation { events?: ObservedEvent[]; @@ -101,6 +102,11 @@ function deferred() { }; } type Gate = ReturnType>; +// The source result with no JSON rendering (D13): its toJSON throws, so every +// JSON rendering of it fails. The driver knows it by identity and observes a +// call that returns it as { unrenderable: true }. +const UNRENDERABLE_VALUE = { toJSON(): never { throw new Error("Controlled unrenderable value"); } }; +type SourceValue = Value | undefined | typeof UNRENDERABLE_VALUE; type Scope = { instance: string; run: (fn: () => T) => T; gate: Gate; lifetime: Promise }; // The first observation member whose encoding differs between two snapshots. @@ -126,13 +132,13 @@ export class BehaviorDriver { private readonly sourceByInvocation = new Map(); private fallbackFailed = false; private adapterReply: { value: AdapterReply } | undefined; - private readonly loaders: Array>> = []; + private readonly loaders: Array>> = []; private readonly sourceErrors: Error[] = []; private readonly timeoutErrors: unknown[] = []; private readonly scopes = new Map(); private readonly effects = { read: new Map(), write: new Map(), dump: new Map(), load: new Map(), policy: new Map() }; - private readonly faults: Faults = { localStorage: false, read: false, write: false, dump: false, load: false, policy: false, observer: false, + private readonly faults: Faults = { localStorage: false, read: false, write: false, dump: false, load: false, policy: false, observer: false, logger: false, holdReads: false, holdWrites: false, holdDumps: false, holdLoads: false, holdPolicies: false }; private runtimePolicy: Policy | null = {}; private wallOffset = 0; @@ -254,12 +260,13 @@ export class BehaviorDriver { }, observeSerialization: (labels, seconds) => { this.record("serialization", { ...labels, seconds }); noop(); }, ...(fixture.shadowHook === false ? {} : { shadowValidation: ({ outcome }: { outcome: string }) => { this.observed.shadow.push(outcome); noop(); } }), - staleRecovery: ({ outcome }) => { this.observed.recovery.push(outcome); noop(); } }, + staleRecovery: ({ outcome }) => { this.observed.recovery.push(outcome); noop(); }, shadowMismatchLogging: (labels) => { this.record("shadowMismatchLogging", labels); noop(); } }, logger: { debug: noop, warn: (message, details) => { if (message === "DialCache shadow validation mismatch" && typeof details === "object" && details !== null) { - this.record("mismatchWarning", details); + this.record("mismatchWarning", warningObservation(details)); } noop(); + if (this.faults.logger) throw new Error("Controlled logger failure"); }, error: noop }, ...this.overrides, }); @@ -305,7 +312,7 @@ export class BehaviorDriver { atMs: performance.now(), budgetMs: sourceBudget }); this.history.push({ event: "sourceStart", id: this.loaders.length, atMs: performance.now() }); if (this.fixture.probeSourceScope) this.observed.sourceScopes.push(cache.isEnabled()); - const gate = deferred(); + const gate = deferred(); this.loaders.push(gate); this.sourceErrors.push(new Error(`Source failure ${this.sourceErrors.length}`)); this.observed.loaders++; @@ -315,6 +322,9 @@ export class BehaviorDriver { return gate.promise; }, { keyType: "id", key: input.key ?? "1", useCase: input.useCase ?? "Behavior", serializer: this.serializer, ...(input.recovery === undefined ? {} : { shouldAttemptStaleRecovery: this.classifier(input.recovery) }), + ...(this.fixture.mismatchDiff === undefined ? {} : { shadowMismatchDiff: () => { + throw new Error("Controlled mismatch diff failure"); + } }), ...(this.fixture.comparator === undefined ? {} : { shadowComparator: () => { this.observed.comparisons++; // Observe elapsed external comparison work without delivering timers. @@ -337,13 +347,14 @@ export class BehaviorDriver { const result = input.scope !== undefined ? this.scope(input.scope).run(execute) : input.outside ? execute() : cache.enable(execute); void result.then( - (value) => { this.observed.calls[index] = { status: "value", value: value === undefined ? { absent: true } : value }; }, + (value) => { this.observed.calls[index] = { status: "value", value: value === undefined ? { absent: true } + : value === UNRENDERABLE_VALUE ? { unrenderable: true } : value as Value }; }, (error: unknown) => { this.observed.calls[index] = { status: "error", error: this.classifyError(error) }; }, ); break; } case "resolve": - this.loader(input.loader).resolve(input.value); + this.loader(input.loader).resolve(input.unrenderable ? UNRENDERABLE_VALUE : input.value); this.history.push({ event: "sourceSettlement", id: input.loader, atMs: performance.now(), outcome: "resolve" }); this.causalHistory.push({ event: "sourceSettlement", id: input.loader, atMs: performance.now(), outcome: "resolve" }); break; @@ -527,3 +538,13 @@ export class BehaviorDriver { return `unexpected:${String(error)}`; } } + +// The warning's fields as logged, each present exactly when the warning +// carries it; the mismatchPaths array crosses the wire as one JSON string so +// every observed field stays a primitive (PORTING.md, mismatchWarning). +function warningObservation(details: object): Record { + const { mismatchPaths, ...fields } = details as Record; + return Object.hasOwn(details, "mismatchPaths") + ? { ...fields, mismatchPathsJson: mismatchPaths === null ? null : JSON.stringify(mismatchPaths) } + : fields; +} diff --git a/typescript/test/formal/feature-profile.ts b/typescript/test/formal/feature-profile.ts index 56d66da5..c3c21823 100644 --- a/typescript/test/formal/feature-profile.ts +++ b/typescript/test/formal/feature-profile.ts @@ -16,6 +16,7 @@ export interface Profile { diagnosticConfigErrors?: boolean; diagnosticFutureOffsets?: boolean; diagnosticInspections?: boolean; + diagnosticWarnings?: boolean; diagnosticAge?: "shadowAge" | "recoveryAge" | "none"; diagnosticUseCase?: string; fixture: Fixture | ((choice: number) => Fixture); diff --git a/typescript/test/prometheus.test.ts b/typescript/test/prometheus.test.ts index 4fe758c6..e1a0dcf9 100644 --- a/typescript/test/prometheus.test.ts +++ b/typescript/test/prometheus.test.ts @@ -16,6 +16,7 @@ import { type CompressionOutcome, type DisabledReason, type MetricErrorKind, + type ShadowMismatchLoggingOutcome, type ShadowValidationOutcome, type StaleRecoveryOutcome, } from "../src/index.js"; @@ -91,6 +92,12 @@ const SHADOW_VALIDATION_OUTCOMES: Readonly timeout: true, dropped: true, }; +const SHADOW_MISMATCH_LOGGING_OUTCOMES: Readonly> = { + logged: true, + hook_error: true, + render_error: true, + logger_error: true, +}; const STALE_RECOVERY_OUTCOMES: Readonly> = { served: true, miss: true, @@ -220,6 +227,12 @@ describe("Prometheus metrics adapter", () => { }, 42, ); + metrics.shadowMismatchLogging({ + cacheNamespace: labels.cacheNamespace, + useCase: labels.useCase, + keyType: labels.keyType, + outcome: "logged", + }); metrics.observeFutureTimestampOffset(labels, 0.007); metrics.compression({ ...labels, outcome: "compressed" }); metrics.observeGet(labels, 0.05); @@ -277,6 +290,10 @@ describe("Prometheus metrics adapter", () => { ["cache_namespace", "use_case", "key_type", "layer", "operation"], TIMER_BUCKETS, ), + counterSchema( + "schema_dialcache_shadow_mismatch_logging_counter", + ["cache_namespace", "use_case", "key_type", "outcome"], + ), counterSchema( "schema_dialcache_shadow_validation_counter", ["cache_namespace", "use_case", "key_type", "outcome"], @@ -508,6 +525,39 @@ describe("Prometheus metrics adapter", () => { ); }); + it("exports every bounded shadow-mismatch-logging outcome without adding cache identity or layer labels", async () => { + const registry = new Registry(); + const metrics = new PrometheusDialCacheMetrics({ registry, prefix: "mismatch_logging_" }); + const labels = { + cacheNamespace: "users", + useCase: "PrometheusShadowMismatchLogging", + keyType: "user_id", + } as const; + const outcomes = Object.keys(SHADOW_MISMATCH_LOGGING_OUTCOMES) as ShadowMismatchLoggingOutcome[]; + + for (const outcome of outcomes) { + metrics.shadowMismatchLogging({ ...labels, outcome }); + } + + for (const outcome of outcomes) { + await expect( + sumMetric(registry, "mismatch_logging_dialcache_shadow_mismatch_logging_counter", { + cache_namespace: labels.cacheNamespace, + use_case: labels.useCase, + key_type: labels.keyType, + outcome, + }), + ).resolves.toBe(1); + } + + const family = ((await registry.getMetricsAsJSON()) as unknown as MetricFamily[]).find( + ({ name }) => name === "mismatch_logging_dialcache_shadow_mismatch_logging_counter", + ); + expect(family?.values.map(({ labels: emitted }) => Object.keys(emitted))).toEqual( + outcomes.map(() => ["cache_namespace", "use_case", "key_type", "outcome"]), + ); + }); + it("exports every bounded stale-recovery outcome without adding cache identity or layer labels", async () => { const registry = new Registry(); const metrics = new PrometheusDialCacheMetrics({ registry, prefix: "stale_" }); @@ -677,6 +727,35 @@ describe("Prometheus metrics adapter", () => { ).resolves.toBe(1); }); + it("reuses a pre-registered current-schema shadow-mismatch-logging collector", async () => { + const registry = new Registry(); + const prefix = "current_mismatch_logging_"; + const metricName = `${prefix}dialcache_shadow_mismatch_logging_counter`; + new Counter({ + name: metricName, + help: "DialCache confirmed shadow-mismatch warning outcomes.", + labelNames: ["cache_namespace", "use_case", "key_type", "outcome"], + registers: [registry], + }); + const metrics = new PrometheusDialCacheMetrics({ registry, prefix }); + + metrics.shadowMismatchLogging({ + cacheNamespace: "users", + useCase: "CurrentMismatchLoggingCollector", + keyType: "user_id", + outcome: "logger_error", + }); + + await expect( + sumMetric(registry, metricName, { + cache_namespace: "users", + use_case: "CurrentMismatchLoggingCollector", + key_type: "user_id", + outcome: "logger_error", + }), + ).resolves.toBe(1); + }); + for (const { schemaPart, register } of incompatibleCollectorCases) { it(`rejects an existing collector with incompatible ${schemaPart} before registering anything`, () => { const registry = new Registry(); diff --git a/typescript/test/shadow-log-json.test.ts b/typescript/test/shadow-log-json.test.ts index 22b17070..f5383931 100644 --- a/typescript/test/shadow-log-json.test.ts +++ b/typescript/test/shadow-log-json.test.ts @@ -1,38 +1,89 @@ -import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; + +import { describe, expect, it, vi } from "vitest"; import { SHADOW_LOG_KEY_MAX_BYTES, SHADOW_LOG_TRUNCATION_MARKER, SHADOW_LOG_VALUE_MAX_BYTES, - previewShadowLogJson, + SHADOW_MISMATCH_PATH_MAX_BYTES, + diffShadowMismatchJson, + failedShadowMismatchLogFields, previewShadowLogKey, - shadowMismatchLogDetails, + renderShadowMismatchLog, + type JsonValue, + type ShadowMismatchJsonEntry, + type ShadowMismatchLogPlan, } from "../src/internal/shadow-log-json.js"; +const VALUE_ONLY: ShadowMismatchLogPlan = { value: true, diff: "off" }; +const PATHS_ONLY: ShadowMismatchLogPlan = { value: false, diff: "paths" }; +const VALUES_ONLY: ShadowMismatchLogPlan = { value: false, diff: "values" }; +const EVERYTHING: ShadowMismatchLogPlan = { value: true, diff: "values" }; + +// The built-in walk's portable vectors (W10). An omitted cached or source +// member is an absent side (D13), TypeScript undefined; the group also pins +// the examples docs/shadow-validation.md shows. +interface MismatchPathVector { + readonly name: string; + readonly cached?: JsonValue; + readonly source?: JsonValue; + readonly entries: readonly ShadowMismatchJsonEntry[]; + readonly count: number; +} + +const MISMATCH_PATH_VECTORS = (JSON.parse( + readFileSync(new URL("../../formal/catalogs/protocol-vectors.json", import.meta.url), "utf8"), +) as { readonly mismatchPathVectors: readonly MismatchPathVector[] }).mismatchPathVectors; +const PRESENT_MISMATCH_PATH_VECTORS = MISMATCH_PATH_VECTORS + .filter((vector) => Object.hasOwn(vector, "cached") && Object.hasOwn(vector, "source")) + .map((vector) => ({ ...vector, cached: vector.cached!, source: vector.source! })); + +const CACHED_USER = { id: "123", email: "old@example.com" }; +const SOURCE_USER = { id: "123", email: "new@example.com" }; +const CACHED_USER_JSON = '{"id":"123","email":"old@example.com"}'; +const SOURCE_USER_JSON = '{"id":"123","email":"new@example.com"}'; +const FAILED_DIFF = { mismatchPaths: null, mismatchPathCount: null, diffJson: null } as const; + +function circular(): unknown { + const value: { self?: unknown } = {}; + value.self = value; + return value; +} + +function nestMembers(depth: number, leaf: JsonValue): JsonValue { + let value = leaf; + for (let level = 0; level < depth; level += 1) { + value = { n: value }; + } + return value; +} + describe("shadow mismatch log JSON", () => { it("uses native JSON for the values supplied to the comparator", () => { - expect(shadowMismatchLogDetails( - "urn:user_id:123#GetUser", + expect(previewShadowLogKey("urn:user_id:123#GetUser")).toBe("urn:user_id:123#GetUser"); + expect(renderShadowMismatchLog( + VALUE_ONLY, { id: "123", updatedAt: new Date("2026-07-31T00:00:00.000Z") }, { id: "123", updatedAt: new Date("2026-08-01T00:00:00.000Z") }, - )).toEqual({ - cacheKey: "urn:user_id:123#GetUser", - cachedValueJson: '{"id":"123","updatedAt":"2026-07-31T00:00:00.000Z"}', - sourceValueJson: '{"id":"123","updatedAt":"2026-08-01T00:00:00.000Z"}', + )).toStrictEqual({ + fields: { + cachedValueJson: '{"id":"123","updatedAt":"2026-07-31T00:00:00.000Z"}', + sourceValueJson: '{"id":"123","updatedAt":"2026-08-01T00:00:00.000Z"}', + }, + failure: null, }); }); it("returns null independently for values that native JSON cannot serialize", () => { - const circular: { self?: unknown } = {}; - circular.self = circular; - - expect(shadowMismatchLogDetails("key", circular, { version: 2 })).toEqual({ - cacheKey: "key", - cachedValueJson: null, - sourceValueJson: '{"version":2}', + expect(renderShadowMismatchLog(VALUE_ONLY, circular(), { version: 2 })).toStrictEqual({ + fields: { cachedValueJson: null, sourceValueJson: '{"version":2}' }, + failure: "render_error", + }); + expect(renderShadowMismatchLog(VALUE_ONLY, 1n, undefined)).toStrictEqual({ + fields: { cachedValueJson: null, sourceValueJson: null }, + failure: "render_error", }); - expect(previewShadowLogJson(1n)).toBeNull(); - expect(previewShadowLogJson(undefined)).toBeNull(); }); it("catches user JSON hook failures", () => { @@ -42,21 +93,592 @@ describe("shadow mismatch log JSON", () => { }, }; - expect(previewShadowLogJson(value)).toBeNull(); + expect(renderShadowMismatchLog(VALUE_ONLY, value, { version: 2 })).toStrictEqual({ + fields: { cachedValueJson: null, sourceValueJson: '{"version":2}' }, + failure: "render_error", + }); }); it("byte-clamps keys and JSON without splitting UTF-8 sequences", () => { const key = `${"k".repeat(SHADOW_LOG_KEY_MAX_BYTES)}🙂`; const value = { text: "🙂".repeat(SHADOW_LOG_VALUE_MAX_BYTES) }; const keyPreview = previewShadowLogKey(key); - const valuePreview = previewShadowLogJson(value); + const { fields } = renderShadowMismatchLog(VALUE_ONLY, value, value); + const valuePreview = fields.cachedValueJson; - expect(valuePreview).not.toBeNull(); + expect(typeof valuePreview).toBe("string"); expect(Buffer.byteLength(keyPreview)).toBeLessThanOrEqual(SHADOW_LOG_KEY_MAX_BYTES); expect(Buffer.byteLength(valuePreview!)).toBeLessThanOrEqual(SHADOW_LOG_VALUE_MAX_BYTES); expect(keyPreview.endsWith(SHADOW_LOG_TRUNCATION_MARKER)).toBe(true); expect(valuePreview!.endsWith(SHADOW_LOG_TRUNCATION_MARKER)).toBe(true); - expect(keyPreview).not.toContain("\uFFFD"); - expect(valuePreview).not.toContain("\uFFFD"); + expect(keyPreview).not.toContain("�"); + expect(valuePreview).not.toContain("�"); + expect(fields.sourceValueJson).toBe(valuePreview); + }); + + describe("built-in mismatch paths", () => { + it.each(PRESENT_MISMATCH_PATH_VECTORS)("computes the expected entries for $name", ({ cached, source, entries, count }) => { + expect(diffShadowMismatchJson(cached, source)).toStrictEqual({ entries, count }); + }); + + // An absent side reaches the walk only through rendering, as undefined. + it.each(MISMATCH_PATH_VECTORS)("renders the expected diff fields for $name", ({ cached, source, entries, count }) => { + const { fields, failure } = renderShadowMismatchLog(VALUES_ONLY, cached, source); + + expect(failure).toBeNull(); + expect(fields.mismatchPaths).toStrictEqual(entries.map((entry) => entry.path)); + expect(fields.mismatchPathCount).toBe(count); + expect(JSON.parse(fields.diffJson!)).toStrictEqual(entries); + expect(renderShadowMismatchLog(PATHS_ONLY, cached, source)).toStrictEqual({ + fields: { mismatchPaths: fields.mismatchPaths, mismatchPathCount: count }, + failure: null, + }); + }); + + it("walks own members whose names collide with Object.prototype", () => { + // Parsed rather than written as literals: a "__proto__" key in an object + // literal sets the prototype instead of creating a member. + const cached = JSON.parse('{"constructor":1,"__proto__":{"a":1}}') as JsonValue; + const source = JSON.parse('{"toString":"x","valueOf":null,"__proto__":{"a":2}}') as JsonValue; + const entries = [ + { path: "/__proto__/a", cached: 1, source: 2 }, + { path: "/constructor", cached: 1 }, + { path: "/toString", source: "x" }, + { path: "/valueOf", source: null }, + ]; + + expect(diffShadowMismatchJson(cached, source)).toStrictEqual({ entries, count: 4 }); + const { fields, failure } = renderShadowMismatchLog(VALUES_ONLY, cached, source); + expect(failure).toBeNull(); + expect(fields.mismatchPaths).toStrictEqual(entries.map((entry) => entry.path)); + expect(JSON.parse(fields.diffJson!)).toStrictEqual(entries); + }); + + it("compares only own members in a depth-capped pair", () => { + const nest = (leaf: string): JsonValue => JSON.parse(`${'{"a":'.repeat(32)}${leaf}${"}".repeat(32)}`) as JsonValue; + const cached = nest('{"__proto__":{}}'); + const source = nest('{"b":{}}'); + const entries = [{ path: "/a".repeat(32), cached: JSON.parse('{"__proto__":{}}') as JsonValue, source: { b: {} } }]; + + expect(diffShadowMismatchJson(cached, source)).toStrictEqual({ entries, count: 1 }); + const { fields, failure } = renderShadowMismatchLog(VALUES_ONLY, cached, source); + expect(failure).toBeNull(); + expect(fields.mismatchPaths).toStrictEqual(["/a".repeat(32)]); + expect(JSON.parse(fields.diffJson!)).toStrictEqual(JSON.parse(JSON.stringify(entries))); + }); + + it("cuts the entries at 64 in walk order and counts every difference", () => { + const cached: Record = {}; + const source: Record = {}; + for (let index = 69; index >= 0; index -= 1) { + cached[`m${String(index).padStart(2, "0")}`] = 0; + source[`m${String(index).padStart(2, "0")}`] = 1; + } + const entries = Array.from({ length: 64 }, (_, index) => ({ + path: `/m${String(index).padStart(2, "0")}`, + cached: 0, + source: 1, + })); + + expect(diffShadowMismatchJson(cached, source)).toStrictEqual({ entries, count: 70 }); + expect(renderShadowMismatchLog(PATHS_ONLY, cached, source)).toStrictEqual({ + fields: { mismatchPaths: entries.map((entry) => entry.path), mismatchPathCount: 70 }, + failure: null, + }); + }); + + it("clamps each pointer to 512 bytes without splitting UTF-8 sequences", () => { + const cached = { ["k".repeat(511)]: 1, ["k".repeat(600)]: 1, ["é".repeat(300)]: 1 }; + const source = { ["k".repeat(511)]: 2, ["k".repeat(600)]: 2, ["é".repeat(300)]: 2 }; + + expect(diffShadowMismatchJson(cached, source)).toStrictEqual({ + entries: [ + { path: `/${"k".repeat(511)}`, cached: 1, source: 2 }, + { path: `/${"k".repeat(497)}${SHADOW_LOG_TRUNCATION_MARKER}`, cached: 1, source: 2 }, + { path: `/${"é".repeat(248)}${SHADOW_LOG_TRUNCATION_MARKER}`, cached: 1, source: 2 }, + ], + count: 3, + }); + }); + + it("builds each kept pointer only as far as its clamp reads, however long the member names above it", () => { + // Three 1 MiB member names above 70 differing members. Built in full, + // each of the 64 kept pointers would carry all three names into the + // clamp; even each name cut to the cap would pass the bound below. + const outer = "o".repeat(1 << 20); + const middle = "d".repeat(1 << 20); + const inner = "i".repeat(1 << 20); + const chain = (leaf: number): JsonValue => ({ + [outer]: { + [middle]: { + [inner]: Object.fromEntries( + Array.from({ length: 70 }, (_, index) => [`m${String(index).padStart(2, "0")}`, leaf]), + ), + }, + }, + }); + const encodeInto = vi.spyOn(TextEncoder.prototype, "encodeInto"); + try { + expect(renderShadowMismatchLog(PATHS_ONLY, chain(1), chain(2))).toStrictEqual({ + fields: { + mismatchPaths: Array.from({ length: 64 }, () => `/${"o".repeat(497)}${SHADOW_LOG_TRUNCATION_MARKER}`), + mismatchPathCount: 70, + }, + failure: null, + }); + // At "paths" the clamp encodes nothing but the kept pointers. + const clamped = encodeInto.mock.calls.map(([text]) => text.length); + expect(clamped.length).toBeGreaterThanOrEqual(64); + expect(Math.max(...clamped)).toBeLessThanOrEqual(3 * SHADOW_MISMATCH_PATH_MAX_BYTES + 1); + } finally { + encodeInto.mockRestore(); + } + }); + + it("replaces each side over 1 KiB with its clamped JSON text", () => { + const list = Array.from({ length: 300 }, () => 1000); + const cached = { + accent: "é".repeat(600), + blob: "x".repeat(2000), + edge: "e".repeat(1022), + extra: "z".repeat(1100), + list, + }; + const source = { accent: "a", blob: "y", edge: "f".repeat(1022), list: [1000] }; + + expect(diffShadowMismatchJson(cached, source)).toStrictEqual({ + entries: [ + { path: "/accent", cached: `"${"é".repeat(504)}${SHADOW_LOG_TRUNCATION_MARKER}`, source: "a" }, + { path: "/blob", cached: `"${"x".repeat(1009)}${SHADOW_LOG_TRUNCATION_MARKER}`, source: "y" }, + { path: "/edge", cached: "e".repeat(1022), source: "f".repeat(1022) }, + { path: "/extra", cached: `"${"z".repeat(1009)}${SHADOW_LOG_TRUNCATION_MARKER}` }, + { path: "/list", cached: `${JSON.stringify(list).slice(0, 1010)}${SHADOW_LOG_TRUNCATION_MARKER}`, source: [1000] }, + ], + count: 5, + }); + }); + + it("caps the walk at 32 reference tokens however deep the difference lies", () => { + expect(diffShadowMismatchJson(nestMembers(40, 1), nestMembers(40, 2))).toStrictEqual({ + entries: [{ path: "/n".repeat(32), cached: nestMembers(8, 1), source: nestMembers(8, 2) }], + count: 1, + }); + const clampedSide = `${'{"n":'.repeat(202)}${SHADOW_LOG_TRUNCATION_MARKER}`; + expect(diffShadowMismatchJson(nestMembers(1000, 1), nestMembers(1000, 2))).toStrictEqual({ + entries: [{ path: "/n".repeat(32), cached: clampedSide, source: clampedSide }], + count: 1, + }); + }); + + it("compares equal containers past the depth cap regardless of member order and depth", () => { + let cached: JsonValue = "leaf"; + let source: JsonValue = "leaf"; + for (let level = 0; level < 100_000; level += 1) { + cached = { a: level, next: cached }; + source = { next: source, a: level }; + } + + expect(diffShadowMismatchJson(cached, source)).toStrictEqual({ entries: [], count: 0 }); + }); + }); + + describe("mismatch log rendering", () => { + it.each([ + ["neither leaf", { value: false, diff: "off" }, {}], + ["diff paths only", PATHS_ONLY, { mismatchPaths: ["/email"], mismatchPathCount: 1 }], + ["diff values without value", VALUES_ONLY, { + mismatchPaths: ["/email"], + mismatchPathCount: 1, + diffJson: '[{"path":"/email","cached":"old@example.com","source":"new@example.com"}]', + }], + ["value only", VALUE_ONLY, { cachedValueJson: CACHED_USER_JSON, sourceValueJson: SOURCE_USER_JSON }], + ["value with diff paths", { value: true, diff: "paths" }, { + mismatchPaths: ["/email"], + mismatchPathCount: 1, + cachedValueJson: CACHED_USER_JSON, + sourceValueJson: SOURCE_USER_JSON, + }], + ["value with diff values", EVERYTHING, { + mismatchPaths: ["/email"], + mismatchPathCount: 1, + diffJson: '[{"path":"/email","cached":"old@example.com","source":"new@example.com"}]', + cachedValueJson: CACHED_USER_JSON, + sourceValueJson: SOURCE_USER_JSON, + }], + ] as const)("renders exactly the fields requested by %s", (_name, plan, fields) => { + const content = renderShadowMismatchLog(plan, CACHED_USER, SOURCE_USER); + + expect(content).toStrictEqual({ fields, failure: null }); + expect(Object.keys(content.fields)).toStrictEqual(Object.keys(fields)); + }); + + it("renders no entries for byte-identical renderings", () => { + const noEntries = { mismatchPaths: [], mismatchPathCount: 0, diffJson: "[]" }; + + expect(renderShadowMismatchLog(EVERYTHING, { id: 1, dropped: undefined }, { id: 1 })).toStrictEqual({ + fields: { ...noEntries, cachedValueJson: '{"id":1}', sourceValueJson: '{"id":1}' }, + failure: null, + }); + expect(renderShadowMismatchLog(VALUES_ONLY, new Map([["a", 1]]), {})).toStrictEqual({ + fields: noEntries, + failure: null, + }); + }); + + it("renders no entries for the documented Date row", () => { + // docs/shadow-validation.md: the JSON serializer stored the Date as this + // string, and the source returns the Date itself. + expect(renderShadowMismatchLog( + VALUES_ONLY, + { at: "2026-10-24T00:00:00.000Z" }, + { at: new Date("2026-10-24T00:00:00.000Z") }, + )).toStrictEqual({ + fields: { mismatchPaths: [], mismatchPathCount: 0, diffJson: "[]" }, + failure: null, + }); + }); + + it("reports the source whole at the root for the documented undefined row", () => { + // docs/shadow-validation.md: a cached negative-cache entry against a value. + expect(renderShadowMismatchLog(EVERYTHING, undefined, { plan: "pro" })).toStrictEqual({ + fields: { + mismatchPaths: [""], + mismatchPathCount: 1, + diffJson: '[{"path":"","source":{"plan":"pro"}}]', + cachedValueJson: null, + sourceValueJson: '{"plan":"pro"}', + }, + failure: null, + }); + }); + + it("renders no entries for a Date against its ISO string", () => { + expect(renderShadowMismatchLog( + VALUES_ONLY, + { updatedAt: new Date("2026-07-31T00:00:00.000Z") }, + { updatedAt: "2026-07-31T00:00:00.000Z" }, + )).toStrictEqual({ + fields: { mismatchPaths: [], mismatchPathCount: 0, diffJson: "[]" }, + failure: null, + }); + }); + + it.each([ + ["cycle", circular()], + ["bigint", 1n], + ["throwing toJSON", { toJSON: (): never => { throw new Error("not serializable"); } }], + ["function", () => 1], + ["symbol", Symbol("value")], + ])("nulls the fields fed by a root %s with render_error", (_name, unrenderable) => { + expect(renderShadowMismatchLog(EVERYTHING, unrenderable, { version: 2 })).toStrictEqual({ + fields: { ...FAILED_DIFF, cachedValueJson: null, sourceValueJson: '{"version":2}' }, + failure: "render_error", + }); + expect(renderShadowMismatchLog(EVERYTHING, { version: 2 }, unrenderable)).toStrictEqual({ + fields: { ...FAILED_DIFF, cachedValueJson: '{"version":2}', sourceValueJson: null }, + failure: "render_error", + }); + expect(renderShadowMismatchLog(PATHS_ONLY, { version: 2 }, unrenderable)).toStrictEqual({ + fields: { mismatchPaths: null, mismatchPathCount: null }, + failure: "render_error", + }); + }); + + it("reports an absent root as one side at the root pointer, not as a render failure", () => { + expect(renderShadowMismatchLog(EVERYTHING, undefined, { version: 2 })).toStrictEqual({ + fields: { + mismatchPaths: [""], + mismatchPathCount: 1, + diffJson: '[{"path":"","source":{"version":2}}]', + cachedValueJson: null, + sourceValueJson: '{"version":2}', + }, + failure: null, + }); + expect(renderShadowMismatchLog(EVERYTHING, { version: 2 }, undefined)).toStrictEqual({ + fields: { + mismatchPaths: [""], + mismatchPathCount: 1, + diffJson: '[{"path":"","cached":{"version":2}}]', + cachedValueJson: '{"version":2}', + sourceValueJson: null, + }, + failure: null, + }); + expect(renderShadowMismatchLog(PATHS_ONLY, undefined, null)).toStrictEqual({ + fields: { mismatchPaths: [""], mismatchPathCount: 1 }, + failure: null, + }); + // A stored null is a value, so it stays distinct from absence. + expect(renderShadowMismatchLog(VALUES_ONLY, undefined, null).fields.diffJson).toBe('[{"path":"","source":null}]'); + // Absence beside a side with no rendering is still a render failure. + expect(renderShadowMismatchLog(EVERYTHING, undefined, 1n)).toStrictEqual({ + fields: { ...FAILED_DIFF, cachedValueJson: null, sourceValueJson: null }, + failure: "render_error", + }); + // A custom comparator may call two absent values different; no field differs. + expect(renderShadowMismatchLog(EVERYTHING, undefined, undefined)).toStrictEqual({ + fields: { + mismatchPaths: [], + mismatchPathCount: 0, + diffJson: "[]", + cachedValueJson: null, + sourceValueJson: null, + }, + failure: null, + }); + }); + + it("runs each side's toJSON once for every field it feeds", () => { + let cachedRenders = 0; + let sourceRenders = 0; + const cached = { toJSON: vi.fn(() => ({ email: "a", render: ++cachedRenders })) }; + const source = { toJSON: vi.fn(() => ({ email: "b", render: ++sourceRenders })) }; + + expect(renderShadowMismatchLog(EVERYTHING, cached, source)).toStrictEqual({ + fields: { + mismatchPaths: ["/email"], + mismatchPathCount: 1, + diffJson: '[{"path":"/email","cached":"a","source":"b"}]', + cachedValueJson: '{"email":"a","render":1}', + sourceValueJson: '{"email":"b","render":1}', + }, + failure: null, + }); + expect(cached.toJSON).toHaveBeenCalledOnce(); + expect(source.toJSON).toHaveBeenCalledOnce(); + }); + + it("clamps diffJson to 8 KiB with the truncation marker", () => { + const cached: Record = {}; + const source: Record = {}; + const entries: ShadowMismatchJsonEntry[] = []; + for (let index = 0; index < 20; index += 1) { + const name = `m${String(index).padStart(2, "0")}`; + cached[name] = "x".repeat(500); + source[name] = "y".repeat(500); + entries.push({ path: `/${name}`, cached: "x".repeat(500), source: "y".repeat(500) }); + } + // The 8 KiB bound is the record's, so it is written out rather than + // taken from the implementation's constant. + const diffJson = `${JSON.stringify(entries).slice(0, 8192 - SHADOW_LOG_TRUNCATION_MARKER.length)}${SHADOW_LOG_TRUNCATION_MARKER}`; + + expect(renderShadowMismatchLog(VALUES_ONLY, cached, source)).toStrictEqual({ + fields: { mismatchPaths: entries.map((entry) => entry.path), mismatchPathCount: 20, diffJson }, + failure: null, + }); + expect(Buffer.byteLength(diffJson)).toBe(8192); + }); + + it.each([ + ["diff paths only", PATHS_ONLY], + ["diff values without value", VALUES_ONLY], + ["value only", VALUE_ONLY], + ["value with diff values", EVERYTHING], + ])("fails every field %s requests closed to null, in rendering order", (_name, plan) => { + const rendered = renderShadowMismatchLog(plan, CACHED_USER, SOURCE_USER).fields; + const failed = failedShadowMismatchLogFields(plan); + + expect(failed).toStrictEqual(Object.fromEntries(Object.keys(rendered).map((field) => [field, null]))); + expect(Object.keys(failed)).toStrictEqual(Object.keys(rendered)); + }); + + it("clamps each built-in diffJson side to 1 KiB", () => { + const { fields } = renderShadowMismatchLog(VALUES_ONLY, { blob: "x".repeat(2000) }, { blob: "y" }); + + expect(JSON.parse(fields.diffJson!)).toStrictEqual([ + { path: "/blob", cached: `"${"x".repeat(1009)}${SHADOW_LOG_TRUNCATION_MARKER}`, source: "y" }, + ]); + }); + }); + + describe("mismatch diff hook", () => { + it("takes the entries from the hook and never runs the built-in walk", () => { + const cached = { id: 1 }; + const source = { id: 2 }; + const hook = vi.fn((_cached: unknown, _source: unknown) => [ + { path: "/profile/email", cached: "old@example.com", source: "new@example.com" }, + { path: "/roles" }, + ]); + + expect(renderShadowMismatchLog(EVERYTHING, cached, source, hook)).toStrictEqual({ + fields: { + mismatchPaths: ["/profile/email", "/roles"], + mismatchPathCount: 2, + diffJson: '[{"path":"/profile/email","cached":"old@example.com","source":"new@example.com"},{"path":"/roles"}]', + cachedValueJson: '{"id":1}', + sourceValueJson: '{"id":2}', + }, + failure: null, + }); + expect(hook).toHaveBeenCalledOnce(); + expect(hook.mock.calls[0]![0]).toBe(cached); + expect(hook.mock.calls[0]![1]).toBe(source); + }); + + it("renders neither root side when the hook supplies the entries and value is off", () => { + const cached = { toJSON: vi.fn(() => "cached") }; + const source = { toJSON: vi.fn(() => "source") }; + + expect(renderShadowMismatchLog(VALUES_ONLY, cached, source, () => [])).toStrictEqual({ + fields: { mismatchPaths: [], mismatchPathCount: 0, diffJson: "[]" }, + failure: null, + }); + expect(cached.toJSON).not.toHaveBeenCalled(); + expect(source.toJSON).not.toHaveBeenCalled(); + }); + + it.each([ + ["throws", () => { + throw new Error("hook failed"); + }], + ["returns a rejecting promise", () => Promise.reject(new Error("async hook"))], + ["returns a non-array object", () => ({ path: "/email" })], + ["returns a null member", () => [null]], + ["returns a member whose side getter throws", () => [{ + path: "/email", + get cached(): never { + throw new Error("getter failed"); + }, + }]], + ])("fails closed with hook_error and no fallback when the hook %s", (_name, hook) => { + expect(renderShadowMismatchLog(EVERYTHING, CACHED_USER, SOURCE_USER, hook)).toStrictEqual({ + fields: { ...FAILED_DIFF, cachedValueJson: CACHED_USER_JSON, sourceValueJson: SOURCE_USER_JSON }, + failure: "hook_error", + }); + expect(renderShadowMismatchLog(PATHS_ONLY, CACHED_USER, SOURCE_USER, hook)).toStrictEqual({ + fields: { mismatchPaths: null, mismatchPathCount: null }, + failure: "hook_error", + }); + }); + + it("logs the hook's entries as it returns them and reads only the kept ones", () => { + const hook = (): unknown => [ + { cached: 1 }, + { path: 7 }, + ...Array.from({ length: 62 }, (_, index) => ({ path: `/${index}` })), + null, + ]; + expect(renderShadowMismatchLog(PATHS_ONLY, CACHED_USER, SOURCE_USER, hook)).toStrictEqual({ + fields: { + mismatchPaths: ["undefined", "7", ...Array.from({ length: 62 }, (_, index) => `/${index}`)], + mismatchPathCount: 65, + }, + failure: null, + }); + }); + + it("consumes a rejecting hook result without an unhandled rejection", async () => { + const unhandled = vi.fn(); + process.on("unhandledRejection", unhandled); + try { + const hook = (): unknown => Promise.reject(new Error("async hook")); + expect(renderShadowMismatchLog(PATHS_ONLY, CACHED_USER, SOURCE_USER, hook).failure).toBe("hook_error"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setTimeout(resolve, 10)); + } finally { + process.off("unhandledRejection", unhandled); + } + + expect(unhandled).not.toHaveBeenCalled(); + }); + + it.each([ + ["a bigint", 1n], + ["a cycle", circular()], + ["a function", () => 1], + ["a symbol", Symbol("side")], + ])("keeps the hook paths and nulls diffJson when a hook side is %s", (_name, unrenderable) => { + const hook = (): unknown => [{ path: "/a", cached: 1 }, { path: "/b", source: unrenderable }]; + + expect(renderShadowMismatchLog(VALUES_ONLY, CACHED_USER, SOURCE_USER, hook)).toStrictEqual({ + fields: { mismatchPaths: ["/a", "/b"], mismatchPathCount: 2, diffJson: null }, + failure: "render_error", + }); + expect(renderShadowMismatchLog(PATHS_ONLY, CACHED_USER, SOURCE_USER, hook)).toStrictEqual({ + fields: { mismatchPaths: ["/a", "/b"], mismatchPathCount: 2 }, + failure: null, + }); + }); + + it("stops rendering hook sides at the first side with no rendering", () => { + const later = { toJSON: vi.fn(() => "later") }; + const hook = (): unknown => [{ path: "/a", cached: 1n }, { path: "/b", source: later }]; + + expect(renderShadowMismatchLog(VALUES_ONLY, CACHED_USER, SOURCE_USER, hook)).toStrictEqual({ + fields: { mismatchPaths: ["/a", "/b"], mismatchPathCount: 2, diffJson: null }, + failure: "render_error", + }); + expect(later.toJSON).not.toHaveBeenCalled(); + }); + + it("omits absent hook sides from diffJson and renders present ones as JSON", () => { + const hook = (): unknown => [ + { path: "/a" }, + { path: "/b", cached: 1 }, + { path: "/c", source: null }, + { path: "/d", cached: undefined, source: "x" }, + { path: "/at", cached: new Date("2026-07-31T00:00:00.000Z"), source: { ok: true } }, + ]; + + expect(renderShadowMismatchLog(VALUES_ONLY, CACHED_USER, SOURCE_USER, hook).fields.diffJson).toBe( + '[{"path":"/a"},{"path":"/b","cached":1},{"path":"/c","source":null},{"path":"/d","source":"x"},' + + '{"path":"/at","cached":"2026-07-31T00:00:00.000Z","source":{"ok":true}}]', + ); + }); + + it("clamps hook paths and sides and cuts at 64 with the total count", () => { + const longPath = `/${"k".repeat(600)}`; + const clampedPath = `/${"k".repeat(497)}${SHADOW_LOG_TRUNCATION_MARKER}`; + const hook = (): unknown => [ + { path: longPath, cached: "x".repeat(2000), source: { ok: true } }, + ...Array.from({ length: 69 }, (_, index) => ({ path: `/${index + 1}` })), + ]; + const retained = Array.from({ length: 63 }, (_, index) => `/${index + 1}`); + + const { fields, failure } = renderShadowMismatchLog(VALUES_ONLY, CACHED_USER, SOURCE_USER, hook); + + expect(failure).toBeNull(); + expect(fields.mismatchPaths).toStrictEqual([clampedPath, ...retained]); + expect(fields.mismatchPathCount).toBe(70); + expect(JSON.parse(fields.diffJson!)).toStrictEqual([ + { path: clampedPath, cached: `"${"x".repeat(1009)}${SHADOW_LOG_TRUNCATION_MARKER}`, source: { ok: true } }, + ...retained.map((path) => ({ path })), + ]); + }); + + it("does not call the hook when diff is off", () => { + const hook = vi.fn(() => [{ path: "/email" }]); + + expect(renderShadowMismatchLog(VALUE_ONLY, CACHED_USER, SOURCE_USER, hook)).toStrictEqual({ + fields: { cachedValueJson: CACHED_USER_JSON, sourceValueJson: SOURCE_USER_JSON }, + failure: null, + }); + expect(hook).not.toHaveBeenCalled(); + }); + + it("keeps the hook entries when a root side has no rendering", () => { + expect(renderShadowMismatchLog(EVERYTHING, 1n, SOURCE_USER, () => [{ path: "/a" }])).toStrictEqual({ + fields: { + mismatchPaths: ["/a"], + mismatchPathCount: 1, + diffJson: '[{"path":"/a"}]', + cachedValueJson: null, + sourceValueJson: SOURCE_USER_JSON, + }, + failure: "render_error", + }); + }); + + it("reports hook_error ahead of render_error", () => { + const hook = (): never => { + throw new Error("hook failed"); + }; + + expect(renderShadowMismatchLog(EVERYTHING, circular(), SOURCE_USER, hook)).toStrictEqual({ + fields: { ...FAILED_DIFF, cachedValueJson: null, sourceValueJson: SOURCE_USER_JSON }, + failure: "hook_error", + }); + }); }); });