From ff0a3b5dc860aa8ad7195e1a0999b575c1aa6867 Mon Sep 17 00:00:00 2001 From: leemour Date: Tue, 20 Oct 2026 11:42:58 +0200 Subject: [PATCH] test(projects): rerun the large-document stall on the fixed release after-fix.mjs seeds the same invented 95,200-byte document as run.mjs and runs exact registry Memo 0.79.0 / Core 0.367.0 / Base 0.47.1 under the same guard. The document is now found in one invocation, all five lanes are done and no checkpoint is returned. The result also records that each matching lane reads the whole document again: 476,300 charged bytes, five times the document. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 3 + performance/projects/README.md | 3 +- performance/projects/large-document/README.md | 37 +++++- .../projects/large-document/after-fix.mjs | 121 ++++++++++++++++++ .../large-document/result-after-fix.json | 100 +++++++++++++++ 5 files changed, 262 insertions(+), 2 deletions(-) create mode 100644 performance/projects/large-document/after-fix.mjs create mode 100644 performance/projects/large-document/result-after-fix.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d2a9b..6176629 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ Every entry says what changed as a caller sees it, why, and what to watch for ### Added +- **`performance/projects/large-document/after-fix.mjs`** reruns the stalled large-document reproduction on Memo 0.79.0 and Core + 0.367.0 and asserts progress: the 95,200-byte document is found in one invocation and the run completes. The + result also shows each matching lane reading the whole document again (476,300 charged bytes). - A synthetic extraction-contract study records released SDK quote/schema limits and accepted semantic contradictions using fake completions, without treating structural validation as model quality. diff --git a/performance/projects/README.md b/performance/projects/README.md index f42a4b2..9fa62b7 100644 --- a/performance/projects/README.md +++ b/performance/projects/README.md @@ -4,7 +4,8 @@ exact quotes and size bounds, including accepted contradictory claims and attribution risks. - [Large documents](large-document/README.md): published Memo reproduction of stalled document lanes and - repeated budget charges across two resumes, with a working small-document control. + repeated budget charges across two resumes, with a working small-document control; after the fix (Memo 0.79.0) + the same document is found in one invocation, at five times its size in charged bytes. - [Process and metric definitions](PROCESS.md), [provenance ledger](provenance.json) and [offline verification](verify-provenance.mjs): immutable artifact references, hashes, release provenance diff --git a/performance/projects/large-document/README.md b/performance/projects/large-document/README.md index 6e06bf8..5437321 100644 --- a/performance/projects/large-document/README.md +++ b/performance/projects/large-document/README.md @@ -1,4 +1,4 @@ -# Large project documents: stalled resume reproduction +# Large project documents: stalled resume reproduction and fix ## Why we ran it @@ -58,3 +58,38 @@ evidence, an explicit bounded alternative, or a terminal actionable condition. T no fix and does not alter orchestration, byte budgets, indexing policy or proposal semantics. A page with many smaller matching records may behave differently; this fixture isolates one oversized matching document. It does not claim complete project coverage, full history, semantic extraction or successful recovery. + +## After the fix (2026-10-20) + +**Why:** the reproduction above was handed to the evidence-command owner as BUG-328. Core 0.367.0 +([cli-core#1668](https://github.com/WireCatLabs/cli-core/pull/1668)) changed the reader: a page holds the records that +fit together, a record larger than the page is read alone up to 256 KiB, and one larger still is listed under +`oversized` and skipped. The question: does the released Memo now read this same document and finish? + +**Setup:** Node 24.19.0, exact registry Memo 0.79.0, Core 0.367.0 and Base 0.47.1, the same seeding, guard and +invented document as `run.mjs`. [`after-fix.mjs`](after-fix.mjs) asserts progress instead of the stall: + +```sh +S=$(mktemp -d -p /var/tmp large-doc-sdk-XXXXXX) && HOME=$S XDG_CONFIG_HOME=$S/config XDG_DATA_HOME=$S/data XDG_CACHE_HOME=$S/cache MESSAGING_STORE=$S/store WIRECAT_STATE_DIR=$S/state TMPDIR=$S NPM_CONFIG_USERCONFIG=/dev/null npm install --prefix "$S/prefix" --ignore-scripts --no-audit --no-fund --save-exact @wirecat/cli-memo@0.79.0 @wirecat/cli-core@0.367.0 @wirecat/cli-base@0.47.1 +T=$(mktemp -d -p /var/tmp large-doc-run-XXXXXX) && HOME=$T XDG_CONFIG_HOME=$T/config XDG_DATA_HOME=$T/data XDG_CACHE_HOME=$T/cache MESSAGING_STORE=$T/store WIRECAT_STATE_DIR=$T/state TMPDIR=$T node performance/projects/large-document/after-fix.mjs "$S/prefix" "$T/result.json" +``` + +**Results** ([`result-after-fix.json`](result-after-fix.json)): + +| | Memo 0.76.0 (three invocations) | Memo 0.79.0 (one invocation) | +| --- | --- | --- | +| References found | 0 | 1 (the large document, matched by all five lanes) | +| Units | five `read-limited`, 0 pages | five `done`, 1 page each | +| Checkpoint / `complete` | present / false | none / true | +| Charged bytes | 983,040 | 476,300 | +| Reads | — | 10 | + +The small control still returns one reference for 184 bytes. + +**What it showed:** the stall is gone; the run ends in one invocation without a resume. It also showed waste: +every lane reads the whole document again (two reads per lane: the refused page, then the record alone), and only +then drops it as already found. 476,300 bytes is five times the document. Skipping the read of a record the run +already holds would charge it once. + +**Not measured:** timing, memory, a document above 256 KiB through the CLI (covered by Core tests), many +medium documents on one page, or owner data. diff --git a/performance/projects/large-document/after-fix.mjs b/performance/projects/large-document/after-fix.mjs new file mode 100644 index 0000000..28ba23b --- /dev/null +++ b/performance/projects/large-document/after-fix.mjs @@ -0,0 +1,121 @@ +import assert from "node:assert/strict" +import { execFileSync } from "node:child_process" +import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs" +import { createRequire } from "node:module" +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path" +import { fileURLToPath, pathToFileURL } from "node:url" + +const [prefix, output, worker] = process.argv.slice(2) +assert.ok(prefix && output, "run.mjs INSTALLED_PREFIX OUTPUT_JSON") +const root = realpathSync(process.env.HOME) +assert.ok(root.startsWith("/var/tmp/") && root !== "/var/tmp", "use a fresh /var/tmp HOME") +for (const key of [ + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_CACHE_HOME", + "MESSAGING_STORE", + "WIRECAT_STATE_DIR", + "TMPDIR", +]) { + assert.ok(process.env[key], `set isolated ${key}`) + const child = relative(root, resolve(process.env[key])) + assert.ok(!isAbsolute(child) && child !== ".." && !child.startsWith(`..${sep}`), `${key} must stay inside HOME`) +} +const packages = join(resolve(prefix), "node_modules/@wirecat") +const memo = join(packages, "cli-memo") +const require = createRequire(join(memo, "package.json")) +const artifact = (name, expected) => { + const path = join(packages, name, "package.json") + const { version } = JSON.parse(readFileSync(path, "utf8")) + assert.equal(version, expected) + return { version } +} +const artifacts = { + "cli-memo": artifact("cli-memo", "0.79.0"), + "cli-core": artifact("cli-core", "0.367.0"), + "cli-base": artifact("cli-base", "0.47.1"), +} +const sourceText = "Copper Chronicle CCL Orchard review Planning digest Budget context. ".repeat(1400) +const sourceBytes = Buffer.byteLength(sourceText) +if (worker === "seed") { + assert.ok(!existsSync(process.env.MESSAGING_STORE), "refuse an existing store") + const { openStore } = await import(pathToFileURL(require.resolve("@wirecat/cli-core/store"))) + const store = await openStore({ env: process.env }) + try { + const project = await store.knowledge.addProject({ name: "Copper Chronicle", key: "CCL" }) + for (const alias of ["Orchard review", "Planning digest", "Budget context"]) + await store.knowledge.addVocabularyAlias({ kind: "project", id: project.id }, alias) + await store.knowledge.addProject({ name: "Small Control", key: "SMC" }) + const folder = await store.notes.addFolder({ name: "Synthetic document limit" }) + for (const [path, text] of [ + ["large.md", sourceText], + ["small.md", "Small Control SMC tiny evidence"], + ]) + await store.notes.saveFileNote({ folderId: folder.id, path, title: path, text }) + process.stdout.write(JSON.stringify({ source: `notes:${folder.id}`, source_bytes: sourceBytes })) + } finally { + await store.close() + } +} else { + assert.ok(!existsSync(process.env.MESSAGING_STORE), "refuse an existing store") + const guard = fileURLToPath(new URL("../../documents/drive-shared-store/guard.mjs", import.meta.url)) + const invoke = (entry, args) => { + const stdout = execFileSync(process.execPath, ["--import", guard, entry, ...args], { + env: process.env, + encoding: "utf8", + timeout: 20000, + maxBuffer: 2 * 1024 * 1024, + }) + return { answer: JSON.parse(stdout), output_bytes: Buffer.byteLength(stdout) } + } + const seeded = invoke(fileURLToPath(import.meta.url), [prefix, output, "seed"]).answer + assert.ok(seeded.source_bytes > 65536) + const command = (...args) => invoke(join(memo, "dist/bin/memo.js"), [...args, "--json"]) + const control = command("projects", "evidence", "SMC", "--source", seeded.source) + assert.equal(control.answer.item.items.length, 1) + const result = command("projects", "evidence", "CCL", "--source", seeded.source) + const found = result.answer.item + assert.equal(found.items.length, 1) + assert.equal(found.units.length, 5) + assert.ok(found.units.every((unit) => unit.state === "done" && unit.pages >= 1)) + assert.equal(found.complete, true) + assert.equal(found.checkpoint, null) + assert.deepEqual(found.oversized, []) + const item = found.items[0] + const attempts = [ + { + invocation: 1, + items: found.items.length, + lanes_matching_the_document: item.lanes.length, + matched_again: found.matched_again.length, + returned_excerpt_bytes: Buffer.byteLength(item.text), + around_entries: (item.around ?? []).length, + output_bytes: result.output_bytes, + usage: found.usage, + units: found.units, + checkpoint_present: found.checkpoint !== null, + complete: found.complete, + stopped: found.stopped, + oversized: found.oversized.length, + }, + ] + const lock = JSON.parse(readFileSync(join(resolve(prefix), "package-lock.json"), "utf8")) + for (const [name, item] of Object.entries(artifacts)) { + item.integrity = lock.packages[`node_modules/@wirecat/${name}`].integrity + assert.ok(item.integrity) + } + const report = { + classification: "large-document-read-after-fix", + node: process.version, + artifacts, + source_bytes: sourceBytes, + page_budget_bytes: 65536, + control: { items: control.answer.item.items.length, usage: control.answer.item.usage }, + attempts, + limits: + "Same synthetic fixture as run.mjs, one CLI invocation on the fixed release; no owner data, remote calls, timing benchmark, memory measurement or claim about documents above 256 KiB (listed under oversized, covered by Core tests).", + } + mkdirSync(dirname(resolve(output)), { recursive: true }) + writeFileSync(resolve(output), `${JSON.stringify(report, null, 2)}\n`) + process.stdout.write(`Large document read in one invocation; ${resolve(output)}\n`) +} diff --git a/performance/projects/large-document/result-after-fix.json b/performance/projects/large-document/result-after-fix.json new file mode 100644 index 0000000..891b8c9 --- /dev/null +++ b/performance/projects/large-document/result-after-fix.json @@ -0,0 +1,100 @@ +{ + "classification": "large-document-read-after-fix", + "node": "v24.19.0", + "artifacts": { + "cli-memo": { + "version": "0.79.0", + "integrity": "sha512-vsZq+ehtOMtTDgYIdVHJD7yEqfrSDs010PVIJdXBNuFUj6I45CKlUX5UJXRR+DQyeLAsWL195M2czkajfy1//Q==" + }, + "cli-core": { + "version": "0.367.0", + "integrity": "sha512-H/hbbU0YIj4jLTBPeVBozhj+9cQTalAhoGsdXfRaFsjhYGRJUM3eXFzSqVqB/mqofSCgN/vSpGJEKQgMujyORA==" + }, + "cli-base": { + "version": "0.47.1", + "integrity": "sha512-v6hQV0tKr9FkJenqALpMG9b06H8ZZdDCxZPhpBLAZgr/kKyy1beJBQSCQ4V3jRhRAeM0RQFv1jUlytY/BM0qeA==" + } + }, + "source_bytes": 95200, + "page_budget_bytes": 65536, + "control": { + "items": 1, + "usage": { + "reads": 2, + "read_bytes": 184, + "around_bytes": 0, + "items": 1, + "elapsed_ms": 14 + } + }, + "attempts": [ + { + "invocation": 1, + "items": 1, + "lanes_matching_the_document": 5, + "matched_again": 0, + "returned_excerpt_bytes": 303, + "around_entries": 0, + "output_bytes": 3271, + "usage": { + "reads": 10, + "read_bytes": 476300, + "around_bytes": 0, + "items": 1, + "elapsed_ms": 14 + }, + "units": [ + { + "lane": "l1", + "source": 0, + "after": 1787305314979, + "before": 1792489314979, + "state": "done", + "pages": 1, + "issue": null + }, + { + "lane": "l2", + "source": 0, + "after": 1787305314979, + "before": 1792489314979, + "state": "done", + "pages": 1, + "issue": null + }, + { + "lane": "l3", + "source": 0, + "after": 1787305314979, + "before": 1792489314979, + "state": "done", + "pages": 1, + "issue": null + }, + { + "lane": "l4", + "source": 0, + "after": 1787305314979, + "before": 1792489314979, + "state": "done", + "pages": 1, + "issue": null + }, + { + "lane": "l5", + "source": 0, + "after": 1787305314979, + "before": 1792489314979, + "state": "done", + "pages": 1, + "issue": null + } + ], + "checkpoint_present": false, + "complete": true, + "stopped": null, + "oversized": 0 + } + ], + "limits": "Same synthetic fixture as run.mjs, one CLI invocation on the fixed release; no owner data, remote calls, timing benchmark, memory measurement or claim about documents above 256 KiB (listed under oversized, covered by Core tests)." +}