From 40be6012750be76b8ce5f4e21c1622dfdcf15333 Mon Sep 17 00:00:00 2001 From: leemour Date: Tue, 20 Oct 2026 16:34:00 +0200 Subject: [PATCH] docs(performance): rerun project evidence benchmark on memo 0.82 Same frozen fixture and labels on memo 0.82.0 (a461598) with Core 0.371.0. Numbers are unchanged from the 0.349.0 run; the fixture has no large document, so paging and document paragraphs in around are not exercised. Adds a dated section and result-2026-10-20.json; baseline.json and earlier tables untouched. Co-Authored-By: Claude Opus 5.5 (1M context) --- performance/projects/evidence/README.md | 30 +++ .../projects/evidence/result-2026-10-20.json | 173 ++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 performance/projects/evidence/result-2026-10-20.json diff --git a/performance/projects/evidence/README.md b/performance/projects/evidence/README.md index 4f08b83..df832fb 100644 --- a/performance/projects/evidence/README.md +++ b/performance/projects/evidence/README.md @@ -111,3 +111,33 @@ The numbered points are from the first run; the section above says which of them - Speed at scale: the store has 13 texts. Reads and bytes are counted; time is not. - Any real project or account: no owner data was used. - Interpretation: whether a found text supports a fact, a task or a decision. That is the next stage. + +## 2026-10-20 rerun + +Same machine (AMD Ryzen AI 9 HX 470, Linux 7.0.0-38), Node v24.19.0, same fixture, labels and script. memo 0.82.0 +built from `main` at `a461598` (a detached checkout, `pnpm build`), with `@wirecat/cli-core` 0.371.0. The store is in a +fresh temporary folder. The run exited 0; its output is [`result-2026-10-20.json`](result-2026-10-20.json). + +| Run | Found | Precision | Precision, project items | Recall, all relevant | Recall, reachable | Missed | +|---|---:|---:|---:|---:|---:|---| +| Fresh store, `--no-catch-up`, 60 days | 6 | 0.67 | 0.75 | 0.50 | 0.67 | reply, mail, document, own note | +| Default, 60 days | 9 | 0.67 | 0.71 | 0.75 | 1.00 | reply, own note | +| Default, 365 days | 10 | 0.70 | 0.75 | 0.78 | 1.00 | reply, own note | + +- The default 60-day run took 15 reads (5 searches × 3 sources), 720 bytes of text plus 493 bytes of context. The + index step reported `index: updated` (4 mail and 3 document rows). +- With `--max-pages 2` the run continued 7 times with `--cursor` and returned the same 9 references, with no + duplicates. +- `--source all` read 3 sources and listed the own notes as unread. + +What changed since "After the fixes": Core 0.349.0 to 0.371.0 and memo 0.72/0.328 to 0.82.0, which bring paging of +large documents and document paragraphs in `around`. Nothing moved: every number in the table, the reads, the bytes +and the references found are the same as before, and the two missed texts are the same. + +That is expected, not evidence the new code works. The two imported documents in the fixture are a few lines long, so +they fit in one page and paging never starts. The context of a message hit holds messages, and no document hit has +paragraphs worth showing. The rerun shows only that the changes did not break or shift the small case. + +Not measured in this rerun: a large document (paging), the paragraphs around a document hit, time or speed, any +real project, and whether the context is enough to read a hit. The fixture and labels were not changed to cover them; +a case for them would need a new labelled fixture, kept apart from the frozen baseline. diff --git a/performance/projects/evidence/result-2026-10-20.json b/performance/projects/evidence/result-2026-10-20.json new file mode 100644 index 0000000..aa07638 --- /dev/null +++ b/performance/projects/evidence/result-2026-10-20.json @@ -0,0 +1,173 @@ +{ + "memo": "0.82.0", + "node": "v24.19.0", + "searches": [ + "Harbour Bureau (project)", + "HBB (key)", + "Харбор (alias)", + "bank tracker (keyword)", + "Rita Example (person)" + ], + "fresh_store_no_catch_up": { + "found": 6, + "relevant_found": 4, + "irrelevant_found": [ + "t-alias-noise", + "t-rita-garden" + ], + "precision": 0.67, + "precision_project_context": 0.75, + "recall_all": 0.5, + "recall_reachable": 0.67, + "missed": [ + "t-reply", + "m-tracker", + "d-method", + "n-own" + ], + "in_context_only": [ + "t-reply", + "t-old" + ], + "person_only_ok": true, + "two_searches_ok": false, + "document_time": null, + "items": [ + "t-name project l1", + "t-alias-noise project l2", + "t-alias-latin project l2", + "t-alias-cyrillic project l3", + "t-rita-garden person-only l5", + "t-rita-dm person-only l5" + ], + "units_index_pending": 10, + "units": 15 + }, + "sixty_days": { + "found": 9, + "relevant_found": 6, + "irrelevant_found": [ + "t-alias-noise", + "m-newsletter", + "t-rita-garden" + ], + "precision": 0.67, + "precision_project_context": 0.71, + "recall_all": 0.75, + "recall_reachable": 1, + "missed": [ + "t-reply", + "n-own" + ], + "in_context_only": [ + "t-reply", + "t-old" + ], + "person_only_ok": true, + "two_searches_ok": true, + "document_time": "stored-update", + "items": [ + "t-name project l1", + "d-method project l1", + "t-alias-noise project l2", + "t-alias-latin project l2", + "m-newsletter project l2", + "m-tracker project l2+l4", + "t-alias-cyrillic project l3", + "t-rita-garden person-only l5", + "t-rita-dm person-only l5" + ], + "reads": 15, + "read_bytes": 720, + "around_bytes": 493, + "index": { + "state": "updated", + "mail": 4, + "documents": 3, + "code": null + }, + "units": 15 + }, + "year": { + "found": 10, + "relevant_found": 7, + "irrelevant_found": [ + "t-alias-noise", + "m-newsletter", + "t-rita-garden" + ], + "precision": 0.7, + "precision_project_context": 0.75, + "recall_all": 0.78, + "recall_reachable": 1, + "missed": [ + "t-reply", + "n-own" + ], + "in_context_only": [ + "t-reply" + ], + "person_only_ok": true, + "two_searches_ok": true, + "document_time": "stored-update", + "items": [ + "t-name project l1", + "d-method project l1", + "t-alias-noise project l2", + "t-alias-latin project l2", + "m-newsletter project l2", + "m-tracker project l2+l4", + "t-alias-cyrillic project l3", + "t-rita-garden person-only l5", + "t-rita-dm person-only l5", + "t-old project l1" + ] + }, + "key_as_search": { + "found": 9, + "relevant_found": 6, + "irrelevant_found": [ + "t-alias-noise", + "m-newsletter", + "t-rita-garden" + ], + "precision": 0.67, + "precision_project_context": 0.71, + "recall_all": 0.75, + "recall_reachable": 1, + "missed": [ + "t-reply", + "n-own" + ], + "in_context_only": [ + "t-reply", + "t-old" + ], + "person_only_ok": true, + "two_searches_ok": true, + "document_time": "stored-update", + "items": [ + "t-name project l1", + "d-method project l1", + "t-alias-noise project l2", + "t-alias-latin project l2", + "m-newsletter project l2", + "m-tracker project l2+l4", + "t-alias-cyrillic project l3", + "t-rita-garden person-only l5", + "t-rita-dm person-only l5" + ] + }, + "all_sources": { + "sources": 3, + "unread": [ + "notes" + ] + }, + "resume_two_pages": { + "invocations": 8, + "references": 9, + "duplicates": 0, + "same_as_one_run": true + } +}