diff --git a/CHANGELOG.md b/CHANGELOG.md index b8efdc4e1..880c8ed19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ shape of this file. ## Unreleased +### Fixed + +- **Project gathering reads a large document once, not once per matching lane.** A record too large to share a + page is read alone; when another lane had already returned it, it was still read in full again before being + dropped, so a 95 KB document matched by five lanes charged about 476 KB. It is now skipped without reading. + ## 0.368.0 — 20.10.2026 ### Fixed diff --git a/docs/dev/PROJECT-GATHERING.md b/docs/dev/PROJECT-GATHERING.md index fc8d4fd21..d8545db03 100644 --- a/docs/dev/PROJECT-GATHERING.md +++ b/docs/dev/PROJECT-GATHERING.md @@ -83,8 +83,8 @@ the invocation. ## Units, coverage and resume -A page holds the records that fit `maxPageBytes` together. A record larger than that is read alone, with -up to 256 KiB of the remaining `maxReadBytes`. A record above 256 KiB is not read: `oversized` lists its `ref`, +A page holds the records that fit `maxPageBytes` together. A record larger than that is read alone, once per run however many lanes +match it, with up to 256 KiB of the remaining `maxReadBytes`. A record above 256 KiB is not read: `oversized` lists its `ref`, `bytes`, the lanes that matched it and its source, and the unit moves past it. Open it by reference instead. Every unit reports `state`: `pending`, `partial`, `done`, `unavailable`, `failed` or `read-limited` (the diff --git a/src/services/project-gathering.test.ts b/src/services/project-gathering.test.ts index 6c5a8270b..191e07753 100644 --- a/src/services/project-gathering.test.ts +++ b/src/services/project-gathering.test.ts @@ -383,6 +383,22 @@ it("reads documents too large to share a page one at a time and moves past one t expect(result.usage.readBytes).toBeLessThan(400000) }) +it("reads a large document once when several lanes match it", async () => { + const f = await fixture() + const { note } = await f.store.notes.saveFileNote({ + folderId: f.folder.id, + path: "large.md", + title: "large", + text: `Harbour launch plan ${"x".repeat(95200)}`, + }) + drainNoteIndex(f.db, stemmerCache()) + const request = { ...f.request, sources: f.sources.filter((s) => s.kind === "document") } + const result = await gatherProjectEvidence({ evidence: f.port }, request) + expect(result.items.find((i) => i.ref === `document:${note.id}`)?.lanes).toEqual(["l1", "l2"]) + expect(result.usage.readBytes).toBeGreaterThan(95200) + expect(result.usage.readBytes).toBeLessThan(2 * 95200) +}) + it("reads a large document on resume when the read budget ran out before its own read", async () => { const f = await fixture() const { note } = await f.store.notes.saveFileNote({ diff --git a/src/services/project-gathering.ts b/src/services/project-gathering.ts index 07ddc2f73..27c460551 100644 --- a/src/services/project-gathering.ts +++ b/src/services/project-gathering.ts @@ -128,19 +128,22 @@ export const gatherProjectEvidence = async (ports: ProjectGatheringPorts, reques let stopped: ProjectGatheringStop = null const open = (u: ProjectUnitProgress) => !terminal.has(u.state) && u.state !== "read-limited" - const take = (item: ContextEvidenceItem, lane: ProjectLane, kind: ContextSource["kind"], source: number) => { - const known = byRef.get(item.ref) + /** Records another lane finding a reference this run already returned; true when it had. */ + const again = (ref: string, lane: ProjectLane) => { + const known = byRef.get(ref) if (known) { if (!known.lanes.includes(lane.id)) known.lanes.push(lane.id) if (!lane.kinds.every((k) => k === "person")) known.context = "project" - return - } - const key = seenKey(item.ref) - if (seen.has(key)) { - matchedAgain.set(item.ref, (matchedAgain.get(item.ref) ?? new Set()).add(lane.id)) - return + return true } - seen.add(key) + if (!seen.has(seenKey(ref))) return false + matchedAgain.set(ref, (matchedAgain.get(ref) ?? new Set()).add(lane.id)) + return true + } + + const take = (item: ContextEvidenceItem, lane: ProjectLane, kind: ContextSource["kind"], source: number) => { + if (again(item.ref, lane)) return + seen.add(seenKey(item.ref)) const short = excerpt(item.text, lane.query, 300, { literal: true }) const fingerprint = contextEvidenceFingerprint(item) const entry: ProjectEvidenceItem = { @@ -215,7 +218,8 @@ export const gatherProjectEvidence = async (ports: ProjectGatheringPorts, reques } try { let page = await readPage(Math.min(plan.budgets.maxPageBytes, plan.budgets.maxReadBytes - usage.readBytes), 20) - const big = page.oversized + // A large record another lane already returned is not read again; its page already moves past it. + const big = page.oversized && !again(page.oversized.ref, unit.lane) ? page.oversized : undefined if (big && big.bytes <= CONTEXT_MAX_READ_BYTES) { if (big.bytes > plan.budgets.maxReadBytes - usage.readBytes || usage.reads >= plan.budgets.maxReads) { state.state = "read-limited"