From 5d33a0de5517bc924d526da04d78056554701190 Mon Sep 17 00:00:00 2001 From: leemour Date: Tue, 20 Oct 2026 12:20:10 +0200 Subject: [PATCH] test(projects): retain aggregate installed pagination observation --- performance/projects/PROCESS.md | 4 + performance/projects/README.md | 4 + .../projects/installed-observation/README.md | 72 +++++++++++++++++ .../check-projection.mjs | 65 ++++++++++++++++ .../installed-observation/project-totals.mjs | 67 ++++++++++++++++ .../installed-observation/totals.json | 78 +++++++++++++++++++ 6 files changed, 290 insertions(+) create mode 100644 performance/projects/installed-observation/README.md create mode 100644 performance/projects/installed-observation/check-projection.mjs create mode 100644 performance/projects/installed-observation/project-totals.mjs create mode 100644 performance/projects/installed-observation/totals.json diff --git a/performance/projects/PROCESS.md b/performance/projects/PROCESS.md index 509915a..db64d29 100644 --- a/performance/projects/PROCESS.md +++ b/performance/projects/PROCESS.md @@ -12,6 +12,10 @@ retained bytes; it does not run the applications or establish that a historical ## Existing evidence and its limits +The [installed post-fix observation](installed-observation/README.md) is a separate mutable-archive +measurement. It publishes totals through an allowlisted projection; cases, source references and +per-item rows remain private. Do not combine it with the frozen synthetic precision/recall scores. + | Study | What is measured | Runtime provenance | What it does not establish | | --- | --- | --- | --- | | [Initial evidence](https://github.com/WireCatLabs/cli-testing/tree/2c821aac468c280a24e0d3a7e806e76a8adc67ca/performance/projects/evidence) | Primary-hit retrieval against 13 invented labels, scoped windows and continuation | Reported built Memo 0.72.0 source 65822ca, Core 0.345.0; no retained build hash/lock | Published-package identity, live-source recall or interpretation | diff --git a/performance/projects/README.md b/performance/projects/README.md index 9fa62b7..454508f 100644 --- a/performance/projects/README.md +++ b/performance/projects/README.md @@ -1,5 +1,9 @@ # Projects +- [Installed observation](installed-observation/README.md): allowlisted totals from a private fixed-window + gathering rerun after the document pagination fix; finished pagination remains separate from archive + completeness, relevance and interpretation. + - [Extraction contract](extraction-contract/README.md): fake-completion checks of released SDK schema, exact quotes and size bounds, including accepted contradictory claims and attribution risks. diff --git a/performance/projects/installed-observation/README.md b/performance/projects/installed-observation/README.md new file mode 100644 index 0000000..2ddba1e --- /dev/null +++ b/performance/projects/installed-observation/README.md @@ -0,0 +1,72 @@ +# Installed project gathering after the document pagination fix + +A local project gathering run previously stopped on large documents while continuing to return +checkpoints and charge reserved bytes. This observation asks whether the published fix finishes +the same bounded request and exposes useful document evidence. It informs whether to continue +with source review; it does not score the real project's answers. + +## Setup and reproduction + +Linux x64, Node 24.19.0, installed Memo 0.79.0, Core 0.367.0 and Base 0.47.1. The Memo registry +package matches the release's validated artifact (SHA-256 +`6e1864b1dfbf9f28dd891884ba06eaad5736ac44f30b5e155c957ded8a07ecc5`). Packaged and installed +CLI/real stdio MCP smoke checks passed with invented data and temporary state before the live read. + +The private case reuses the earlier explicit start and exclusive end, aliases, questions and +100-item/96-read budgets, allowing at most four CLI invocations per phase. Phases read the existing +index, perform normal catch-up, then repeat. Before/after observed runtime JavaScript and manifests +match. No source fetching, proposal creation, reset or model call is part of these timed phases. + +The executed case, installed-tool runner, software observations and result stay in cli-private under +`tools/context-gathering/` and `reports/`. The result's checksum identifies those retained bytes; +it cannot reproduce a mutable archive. Public synthetic reproduction remains in +[large documents](../large-document/README.md). + +The one-off projection check uses invented private-label/body sentinels, refuses invalid version +and token metadata, and verifies that distinct paragraphs sharing a reference retain their bytes: + +```sh +T=$(mktemp -d -p /var/tmp) && HOME=$T XDG_CONFIG_HOME=$T/c XDG_DATA_HOME=$T/d XDG_CACHE_HOME=$T/k MESSAGING_STORE=$T/store WIRECAT_STATE_DIR=$T/s TMPDIR=$T node performance/projects/installed-observation/check-projection.mjs +``` + +Generate the committed totals from the retained private result, offline: + +```sh +T=$(mktemp -d -p /var/tmp) && HOME=$T XDG_CONFIG_HOME=$T/c XDG_DATA_HOME=$T/d XDG_CACHE_HOME=$T/k MESSAGING_STORE=$T/store WIRECAT_STATE_DIR=$T/s TMPDIR=$T node performance/projects/installed-observation/project-totals.mjs /PATH/TO/PRIVATE-MEASUREMENT.json +``` + +The public script is an allowlisted aggregate projection. It never opens a store or launches an +application. Do not publish the private input: it contains source/account references and query +metadata. The private report is `2026-10-20-installed-project-workflow.md`. + +## Results + +| Phase | CLI calls | Wall time | Direct refs | Distinct combined refs | Pagination | +| --- | ---: | ---: | ---: | ---: | --- | +| Existing index | 1 | 1,023.13 ms | 67 | 144 | Finished | +| Normal catch-up | 1 | 1,411.83 ms | 67 | 144 | Finished | +| Normal repeat | 1 | 609.80 ms | 67 | 144 | Finished | + +Every phase returned 41 messages, 25 documents and one meeting cue, with 119 distinct surrounding +references. Direct and surrounding sets overlap. No continuation or no-progress stop was needed; +no record was excluded as oversized. All three phases returned the same direct references and +excerpt hashes. A bounded project-review snapshot was unchanged, not a whole-store diff. + +Each phase charged 1,285,955 primary budget bytes and 35,434 surrounding bytes. Returned direct +excerpts contained 21,481 bytes; 196 distinct surrounding excerpts contained 29,171 bytes. Different +paragraphs can share a document reference, so excerpt bytes are deduplicated by reference **and** +text hash. Charged bytes are reservations, not actual I/O or text shown to an agent. The released +cross-lane large-document reread inefficiency remains separately measured by the synthetic study. + +## What this establishes and leaves open + +The previous fixed-window observation returned 45 direct/122 combined references and stopped +after a stalled continuation. The new one retains those references and their excerpt hashes and +adds 22 document references. The aliases/window are unchanged, but an additional account is now +registered and the archive is mutable. This is no controlled speedup or exhaustive recall claim. + +Finished pagination leaves archive completeness unknown. Native provider discovery, missing source +imports, reading complete relevant passages, project attribution, contradictory versions and +proposal review remain separate work. Agent reading/reasoning, manual tool calls, remote imports, +model accuracy and token costs were not timed. The repeated phase returns evidence again; it does +not establish an incremental AI-analysis cache. diff --git a/performance/projects/installed-observation/check-projection.mjs b/performance/projects/installed-observation/check-projection.mjs new file mode 100644 index 0000000..5999cbf --- /dev/null +++ b/performance/projects/installed-observation/check-projection.mjs @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import { spawnSync } from "node:child_process" +import { readFileSync, writeFileSync } from "node:fs" +import { join, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const temporary = resolve(process.env.TMPDIR) +assert.ok(temporary.startsWith("/var/tmp/")) +assert.equal(resolve(process.env.HOME), temporary) +const privateLabel = "Private Fixture Project" +const privateRef = "document:999991" +const privateBody = "Invented source body that must stay private" +const fixture = { + memo: "0.79.0", + dependencies: { core: "0.367.0", base: "0.47.1" }, + node: "v24.19.0", + project: privateLabel, + llm_tokens: null, + runs: Array.from({ length: 3 }, () => ({ + unique_refs: 1, + wall_ms: 1, + finished_pagination: true, + stagnation_detected: false, + items: [ + { + ref: privateRef, + kind: "document", + text: privateBody, + text_utf8_bytes: 10, + around: [ + { ref: privateRef, text: privateBody, text_sha256: "a", text_utf8_bytes: 2 }, + { ref: privateRef, text: privateBody, text_sha256: "b", text_utf8_bytes: 3 }, + ], + }, + ], + pages: [ + { scope: { sources: [{ account: privateLabel }] }, oversized: [], usage: { read_bytes: 20, around_bytes: 5 } }, + ], + })), +} +const input = join(temporary, "fixture.json") +const script = fileURLToPath(new URL("./project-totals.mjs", import.meta.url)) +const run = (value) => { + writeFileSync(input, JSON.stringify(value)) + return spawnSync(process.execPath, [script, input], { env: process.env, encoding: "utf8" }) +} +const result = run(fixture) +assert.equal(result.status, 0, result.stderr) +for (const value of [privateLabel, privateRef, privateBody]) assert.ok(!result.stdout.includes(value)) +const summary = JSON.parse(result.stdout) +assert.equal(summary.phases[0].distinct_surrounding_excerpt_bytes, 5) +assert.equal(summary.phases[0].surrounding_references, 1) +assert.equal(summary.phases[0].distinct_surrounding_excerpts, 2) +assert.notEqual(run({ ...fixture, memo: privateLabel }).status, 0) +assert.notEqual(run({ ...fixture, llm_tokens: { body: privateBody } }).status, 0) +assert.equal(readFileSync(input, "utf8").includes(privateBody), true) +process.stdout.write( + `${JSON.stringify({ + metadata_only: true, + paragraphs_sharing_reference_preserved: true, + invalid_version_refused: true, + nonnumeric_token_metadata_refused: true, + owner_data_access: false, + })}\n`, +) diff --git a/performance/projects/installed-observation/project-totals.mjs b/performance/projects/installed-observation/project-totals.mjs new file mode 100644 index 0000000..c2bb9fc --- /dev/null +++ b/performance/projects/installed-observation/project-totals.mjs @@ -0,0 +1,67 @@ +import assert from "node:assert/strict" +import { createHash } from "node:crypto" +import { lstatSync, readFileSync } from "node:fs" + +const [input] = process.argv.slice(2) +assert.ok(input, "project-totals.mjs PRIVATE_MEASUREMENT_JSON") +const stat = lstatSync(input) +assert.ok(stat.isFile() && !stat.isSymbolicLink(), "measurement must be an ordinary file") +assert.ok(stat.size <= 4 * 1024 * 1024, "measurement exceeds 4 MiB") +const bytes = readFileSync(input) +const report = JSON.parse(bytes) +for (const version of [report.memo, report.dependencies?.core, report.dependencies?.base]) + assert.match(version, /^\d+\.\d+\.\d+$/) +assert.match(report.node, /^v\d+\.\d+\.\d+$/) +assert.ok(report.llm_tokens === null || (Number.isSafeInteger(report.llm_tokens) && report.llm_tokens >= 0)) +assert.ok(Array.isArray(report.runs) && report.runs.length === 3) +const sum = (values, select) => values.reduce((total, value) => total + select(value), 0) +const number = (value) => { + assert.ok(Number.isFinite(value) && value >= 0) + return value +} +const phases = report.runs.map((run, index) => { + const direct = new Set(run.items.map((item) => item.ref)) + const around = new Set(run.items.flatMap((item) => item.around.map((one) => one.ref))) + const excerpts = new Map( + run.items.flatMap((item) => item.around.map((one) => [`${one.ref}:${one.text_sha256}`, one])), + ) + assert.equal(direct.size, run.unique_refs) + return { + phase: ["index_as_stored", "default_catch_up", "default_repeat"][index], + invocations: run.pages.length, + wall_ms: number(run.wall_ms), + direct_references: direct.size, + surrounding_references: around.size, + combined_references: new Set([...direct, ...around]).size, + direct_by_kind: Object.fromEntries( + ["message", "document", "meeting"].map((kind) => [kind, run.items.filter((item) => item.kind === kind).length]), + ), + direct_excerpt_bytes: sum(run.items, (item) => number(item.text_utf8_bytes)), + distinct_surrounding_excerpt_bytes: sum([...excerpts.values()], (item) => number(item.text_utf8_bytes)), + distinct_surrounding_excerpts: excerpts.size, + charged_read_budget_bytes: sum(run.pages, (page) => number(page.usage.read_bytes)), + charged_surrounding_budget_bytes: sum(run.pages, (page) => number(page.usage.around_bytes)), + pagination_finished: run.finished_pagination === true, + no_progress_stop: run.stagnation_detected === true, + oversized_references: new Set(run.pages.flatMap((page) => (page.oversized ?? []).map((item) => item.ref))).size, + resolved_sources: run.pages[0].scope.sources.length, + } +}) +process.stdout.write( + `${JSON.stringify( + { + memo: report.memo, + core: report.dependencies.core, + base: report.dependencies.base, + node: report.node, + private_measurement_sha256: createHash("sha256").update(bytes).digest("hex"), + phases, + app_model_calls: 0, + model_tokens: report.llm_tokens, + qualified_as: + "Mutable local archive observation; not complete remote coverage, semantic recall, model quality, agent time, actual I/O or cold-cache speedup.", + }, + null, + 2, + )}\n`, +) diff --git a/performance/projects/installed-observation/totals.json b/performance/projects/installed-observation/totals.json new file mode 100644 index 0000000..79831a7 --- /dev/null +++ b/performance/projects/installed-observation/totals.json @@ -0,0 +1,78 @@ +{ + "memo": "0.79.0", + "core": "0.367.0", + "base": "0.47.1", + "node": "v24.19.0", + "private_measurement_sha256": "d175357208b8f30e04fb255355baaabb4fdb4dbc3911d523ab5cd4c7b67f0bcf", + "phases": [ + { + "phase": "index_as_stored", + "invocations": 1, + "wall_ms": 1023.13, + "direct_references": 67, + "surrounding_references": 119, + "combined_references": 144, + "direct_by_kind": { + "message": 41, + "document": 25, + "meeting": 1 + }, + "direct_excerpt_bytes": 21481, + "distinct_surrounding_excerpt_bytes": 29171, + "distinct_surrounding_excerpts": 196, + "charged_read_budget_bytes": 1285955, + "charged_surrounding_budget_bytes": 35434, + "pagination_finished": true, + "no_progress_stop": false, + "oversized_references": 0, + "resolved_sources": 11 + }, + { + "phase": "default_catch_up", + "invocations": 1, + "wall_ms": 1411.83, + "direct_references": 67, + "surrounding_references": 119, + "combined_references": 144, + "direct_by_kind": { + "message": 41, + "document": 25, + "meeting": 1 + }, + "direct_excerpt_bytes": 21481, + "distinct_surrounding_excerpt_bytes": 29171, + "distinct_surrounding_excerpts": 196, + "charged_read_budget_bytes": 1285955, + "charged_surrounding_budget_bytes": 35434, + "pagination_finished": true, + "no_progress_stop": false, + "oversized_references": 0, + "resolved_sources": 11 + }, + { + "phase": "default_repeat", + "invocations": 1, + "wall_ms": 609.8, + "direct_references": 67, + "surrounding_references": 119, + "combined_references": 144, + "direct_by_kind": { + "message": 41, + "document": 25, + "meeting": 1 + }, + "direct_excerpt_bytes": 21481, + "distinct_surrounding_excerpt_bytes": 29171, + "distinct_surrounding_excerpts": 196, + "charged_read_budget_bytes": 1285955, + "charged_surrounding_budget_bytes": 35434, + "pagination_finished": true, + "no_progress_stop": false, + "oversized_references": 0, + "resolved_sources": 11 + } + ], + "app_model_calls": 0, + "model_tokens": null, + "qualified_as": "Mutable local archive observation; not complete remote coverage, semantic recall, model quality, agent time, actual I/O or cold-cache speedup." +}