diff --git a/architecture.md b/architecture.md index 41a715b92..221423a00 100644 --- a/architecture.md +++ b/architecture.md @@ -7085,6 +7085,19 @@ Batch-local ids keep that set as small as one batch. This bounds the request, not the document: nothing is truncated to make an engine accept it, and the answers come back one per candidate in the order they were asked for. +**An engine bounds the text one render measures as well as its elements.** Every +space-separated part of every text in a render is kept in a cache the engine +refuses from, and that cache holds the same number of them however few elements +carry them — so a batch well inside the element budget still overruns it when +its rows are prose, and one line of a retained reading can hold more parts than +any render measures whatever a batch size is. A batch is therefore bounded by +both, and a line is measured as far as it could still fit its region rather than +to its end: every part after the first is preceded by a space, so a prefix of +more parts than the region has columns is already wider than the region, and +text does not narrow when more of it is appended. The cut that prefix bounds is +the reader's own line, measured as the row it became — a bounded question, not a +shortened document. + An engine that is asked for too much at once does not say so usefully. It reports unbalanced structure for a list whose opens and closes are exactly paired, which is not a capacity refusal — so it is not reclassified as one diff --git a/packages/cli/src/repl/fitting.ts b/packages/cli/src/repl/fitting.ts index 6e3902237..4c5e0c8fb 100644 --- a/packages/cli/src/repl/fitting.ts +++ b/packages/cli/src/repl/fitting.ts @@ -306,6 +306,61 @@ const SEPARATOR = " "; */ const WIDTHS_PER_RENDER = 1024; +/** + * How many space-separated parts one measuring render may carry. + * + * The element budget is not the engine's only one. Every part of every text a + * render measures is kept in a cache the engine refuses from, and that cache + * holds the same number of them however few elements carry them: measured on a + * freshly built engine, one text of 16,384 parts and 1,366 texts of twelve each + * refuse, while 16,383 parts go through either way. A tab and a newline are not + * separators there, a run of spaces produces an empty part that costs what a + * word costs, and two texts spelled the same share one entry. + * + * This is the budget whose refusal the renderer's recovery cannot absorb: a + * replacement instance is given the same cache, so a render that overran it + * overruns the replacement too and the refusal reaches the screen as the error + * that ends it. Half the engine's own figure, which leaves the parts an + * instance is still holding from the render before this one somewhere to sit. + */ +const WORDS_PER_RENDER = 8192; + +/** What one text costs that budget, counted the way the engine counts it. */ +function wordsIn(text: string): number { + let parts = 1; + for (let at = 0; at < text.length; at += 1) { + if (text[at] === " ") { + parts += 1; + } + } + return parts; +} + +/** + * How much of one line could still fit a region, as a length in code units. + * + * A single text is bounded by no batch size, and one line of somebody's output + * can hold more parts than a render measures — a journal of this product holds + * lines of 19,999 — so a line is measured as far as it could still fit and + * never further. That prefix settles the question it is asked: every part after + * the first is preceded by a space, so `room + 2` of them are already wider + * than `room` columns, and text does not narrow when more of it is appended. + * The prefix ends at a space, which is a grapheme boundary of its own. + */ +function windowOf(text: string, room: number): number { + const parts = Math.min(room + 2, WORDS_PER_RENDER); + let seen = 0; + for (let at = 0; at < text.length; at += 1) { + if (text[at] === " ") { + seen += 1; + if (seen === parts) { + return at; + } + } + } + return text.length; +} + /** * Ask the engine how wide each of these texts is. * @@ -324,12 +379,26 @@ function* widths( return Ok([]); } const answers: number[] = []; - for (let from = 0; from < texts.length; from += WIDTHS_PER_RENDER) { - const batch = yield* batchWidths(renderer, size, texts.slice(from, from + WIDTHS_PER_RENDER)); + let from = 0; + while (from < texts.length) { + let to = from; + let words = 0; + while (to < texts.length && to - from < WIDTHS_PER_RENDER) { + const cost = wordsIn(texts[to]); + // A text that outgrows the budget on its own still goes, alone, so the + // engine's refusal names that text rather than this loop never advancing. + if (to > from && words + cost > WORDS_PER_RENDER) { + break; + } + words += cost; + to += 1; + } + const batch = yield* batchWidths(renderer, size, texts.slice(from, to)); if (!batch.ok) { return batch; } answers.push(...batch.value); + from = to; } return Ok(answers); } @@ -679,11 +748,15 @@ function* fitLine( if (line.text.length === 0) { return Ok([piece(0, 0, false, 0)]); } - const whole = yield* widths(renderer, size, [line.text]); - if (!whole.ok) { - return whole; + // Asked of as much of the line as the region could still hold. One row is a + // line the engine measured as fitting, and a line held back from the question + // is longer than the answer it would have given, so it is cut. + const reach = windowOf(line.text, room); + const reaching = yield* widths(renderer, size, [line.text.slice(0, reach)]); + if (!reaching.ok) { + return reaching; } - if (whole.value[0] <= room) { + if (reach === line.text.length && reaching.value[0] <= room) { return Ok([piece(0, line.text.length, false, 0)]); } // Where this line begins, so its continuations begin there too. Bounded well @@ -728,7 +801,10 @@ function* cutAt( rest: string, room: number, ): Operation> { - const stops = graphemeStops(rest); + // Over the boundaries of the prefix that could still fit, which is what keeps + // every candidate inside one render's budget and keeps this from segmenting + // the whole remainder once per row of a very long line. + const stops = graphemeStops(rest.slice(0, windowOf(rest, room))); // Narrow by halves over grapheme boundaries, asking the engine for one batch // of candidates per round rather than one render per candidate. let low = 0; diff --git a/packages/cli/tests/repl-source-reading.test.ts b/packages/cli/tests/repl-source-reading.test.ts index 4b845611a..a8045ab61 100644 --- a/packages/cli/tests/repl-source-reading.test.ts +++ b/packages/cli/tests/repl-source-reading.test.ts @@ -1176,3 +1176,90 @@ describe("S6 a reading longer than one measuring render is still measured whole" })); } }); + +/** + * S7 a reading with more words than one measuring render. + * + * The engine's element budget is not its only one. It keeps every + * space-separated part of every text a render measures in a cache it refuses + * from, and that cache holds 16,384 of them however few elements carry them — + * measured on a freshly built engine, 1,366 texts of twelve parts and one text + * of 16,384 both refuse. A refusal a bigger engine cannot fix is the one the + * renderer's recovery cannot absorb, so it reached the screen as the error that + * ends it. + * + * Both arms are the shape the reported crash had. An entry whose recorded + * output holds one line of 20,000 parts is the real one — a journal of this + * product holds lines of 19,999 — and it refused inside the first measurement + * of that single line, which no batch size bounds. A reading of 1,400 ordinary + * lines is the other half: each line measures on its own, and the pass that + * measures every finished row at once asked for more parts than the cache holds + * even though it asked for fewer elements than the element budget. + */ +/** One line of `count` space-separated parts, which is what the engine counts. */ +function parts(count: number): string { + return Array.from({ length: count }, (_, at) => `${at % 10}`).join(" "); +} + +describe("S7 a reading with more words than one measuring render is still measured whole", () => { + /** One recorded output line, as the reading an entry is shown as. */ + function outputLinesOf(output: string): readonly ReplReadingLine[] { + return entryReading({ + entry: entryOf('', { + terminal: settled("ok", output), + settled: true, + }), + lifecycle: NO_LIFECYCLE, + live: "", + inspected: false, + }).output.lines; + } + + it("recovers one line of 20,000 parts, exactly and in order", () => + withRenderer(function* (renderer) { + const line = parts(20000); + const lines = outputLinesOf(line); + const prepared = yield* prepareReading(renderer, WIDE, lines, 100); + if (!prepared.ok) { + throw prepared.error; + } + + // One line of somebody's output, whole: it was wrapped into many rows and + // nothing was shortened to make the engine accept it. + const whole = recovered(prepared.value); + expect(whole.size).toBe(lines.length); + for (const one of lines) { + expect([one.key, whole.get(one.key)]).toEqual([one.key, one.text]); + } + const rows = prepared.value.rows.filter((one) => one.line === lines[lines.length - 1].key); + expect(rows.length).toBeGreaterThan(1); + + // And every row still fits the region it was cut for, which is the claim a + // width answered for a bounded prefix could break. + const { reservation } = prepared.value; + for (const row of prepared.value.rows) { + expect(row.width).toBeLessThanOrEqual(reservation.content - row.indent); + expect(runText(row.runs)).toBe(row.text); + } + })); + + it("recovers 1,400 ordinary lines whose parts outgrow one render", () => + withRenderer(function* (renderer) { + // Distinct lines, because the engine caches one entry per exact text: a + // thousand copies of one line cost it the parts of a single line and + // would prove nothing about what a batch of them asks for. + const lines = outputLinesOf( + Array.from({ length: 1400 }, (_, at) => `line ${at} ${parts(20)}`).join("\n"), + ); + const prepared = yield* prepareReading(renderer, WIDE, lines, 100); + if (!prepared.ok) { + throw prepared.error; + } + + const whole = recovered(prepared.value); + expect(whole.size).toBe(lines.length); + for (const one of lines) { + expect([one.key, whole.get(one.key)]).toEqual([one.key, one.text]); + } + })); +});