From deb7a89252ee440fc7bc3ee6655076bc4af2751d Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Tue, 3 Nov 2026 19:04:11 +0100 Subject: [PATCH] perf: let metadata and section-pages start on their dependency's first chunk With more than one thread, metadata and section-pages now start on the first chunk of their dependency instead of waiting for all of it, so parsing, metadata and section pages overlap. The generated files are unchanged. section-pages passes each module's entries on as they arrive and adds the chunk pages after the last of them, in module order, as if built from the whole input at once. A module's chunk pages need all of its entries, and a chunk need not hold whole modules, so groupWholeModules regroups streamed entries into batches of whole modules, and rejects a module whose entries come back after another's. Assisted-by: Claude Opus 5.5 Assisted-by: Claude Sonnet 5.5 --- .changeset/stream-metadata.md | 6 + docs/creating-generators.md | 5 +- .../core/src/generators/metadata/index.mjs | 2 + .../core/src/generators/metadata/types.d.ts | 5 +- .../src/utils/__tests__/generators.test.mjs | 92 +++++++++++++ packages/core/src/utils/generators.mjs | 61 +++++++++ .../src/html/__tests__/generate.test.mjs | 21 ++- .../section-pages/__tests__/generate.test.mjs | 125 ++++++++++++++++-- packages/react/src/section-pages/generate.mjs | 47 +++++-- packages/react/src/section-pages/index.mjs | 2 + packages/react/src/section-pages/types.d.ts | 5 +- 11 files changed, 348 insertions(+), 23 deletions(-) create mode 100644 .changeset/stream-metadata.md diff --git a/.changeset/stream-metadata.md b/.changeset/stream-metadata.md new file mode 100644 index 000000000..984011c27 --- /dev/null +++ b/.changeset/stream-metadata.md @@ -0,0 +1,6 @@ +--- +'@doc-kit/core': minor +'@doc-kit/generator-react': minor +--- + +`metadata` and `section-pages` stream their input: with more than one thread, they start on their dependency's first chunk, so parsing, metadata and section pages overlap. The generated files are unchanged. `groupWholeModules`, from `@doc-kit/core/utils/generators.mjs`, regroups streamed entries so that each batch holds whole modules. diff --git a/docs/creating-generators.md b/docs/creating-generators.md index 62aa810d3..a46447404 100644 --- a/docs/creating-generators.md +++ b/docs/creating-generators.md @@ -379,7 +379,10 @@ export async function* generate(input, worker) { - With more than one thread, `input` can be iterated only once, and passing it to `worker.stream` iterates it. If you need it twice, collect it into an array first. -- The chunks are the dependency's own, in the order it yields them. +- The chunks are the dependency's own, in the order it yields them. `metadata`'s + hold whole files, but chunks need not hold whole modules: a generator that + needs all of a module's entries at once should regroup them with + `groupWholeModules` from `@doc-kit/core/utils/generators.mjs`. - Work that needs the whole input, such as sorting or writing one file, has to wait for the last chunk. - Other generators reading the same dependency are not affected: each gets diff --git a/packages/core/src/generators/metadata/index.mjs b/packages/core/src/generators/metadata/index.mjs index fcdd965b7..64747e682 100644 --- a/packages/core/src/generators/metadata/index.mjs +++ b/packages/core/src/generators/metadata/index.mjs @@ -16,6 +16,8 @@ export default { hasParallelProcessor: true, + streamsInput: true, + markdown: { remarkPlugins: [ 'remark-parse', diff --git a/packages/core/src/generators/metadata/types.d.ts b/packages/core/src/generators/metadata/types.d.ts index c84305c37..0e04b1e11 100644 --- a/packages/core/src/generators/metadata/types.d.ts +++ b/packages/core/src/generators/metadata/types.d.ts @@ -3,7 +3,10 @@ import type { Position } from 'unist'; export type Generator = GeneratorMetadata< {}, - Generate, AsyncGenerator>, + Generate< + Array | AsyncIterable>, + AsyncGenerator + >, ProcessChunk> >; diff --git a/packages/core/src/utils/__tests__/generators.test.mjs b/packages/core/src/utils/__tests__/generators.test.mjs index a9b550373..f81eb1b43 100644 --- a/packages/core/src/utils/__tests__/generators.test.mjs +++ b/packages/core/src/utils/__tests__/generators.test.mjs @@ -4,6 +4,7 @@ import { describe, it } from 'node:test'; import { getEntryDescription, groupNodesByModule, + groupWholeModules, getVersionFromSemVer, coerceSemVer, getCompatibleVersions, @@ -87,6 +88,97 @@ describe('groupNodesByModule', () => { }); }); +describe('groupWholeModules', () => { + /** + * Collects the batches regrouped from the given input, each entry as its + * module and number. + * + * @param {Array<{ api: string }> | AsyncIterable>} input + * @returns {Promise} + */ + const collect = async input => { + const output = []; + + for await (const batch of groupWholeModules(input)) { + output.push(batch.map(({ api, n }) => `${api}${n}`)); + } + + return output; + }; + + /** + * Regroups the given batches, streamed in, and collects the batches it + * yields. + * + * @param {Array>} batches + * @returns {Promise} + */ + const regroup = batches => { + async function* input() { + yield* batches; + } + + return collect(input()); + }; + + /** + * An entry of a module, told apart from the module's others by its number. + * + * @param {string} api + * @param {number} n + * @returns {{ api: string, n: number }} + */ + const entry = (api, n) => ({ api, n }); + + it('passes the entries on in the same order, each batch ending once its last module is whole', async () => { + const output = await regroup([ + [entry('a', 1), entry('a', 2), entry('b', 1)], + [entry('c', 1)], + ]); + + assert.deepEqual(output, [['a1', 'a2'], ['b1'], ['c1']]); + }); + + it('holds back a module that continues in the next batch', async () => { + const output = await regroup([ + [entry('a', 1), entry('b', 1)], + [entry('b', 2), entry('b', 3)], + [entry('b', 4), entry('c', 1)], + ]); + + assert.deepEqual(output, [['a1'], ['b1', 'b2', 'b3', 'b4'], ['c1']]); + }); + + it('skips empty batches', async () => { + const output = await regroup([[], [entry('a', 1)], []]); + + assert.deepEqual(output, [['a1']]); + }); + + it('takes entries given as one array as a single batch', async () => { + const output = await collect([entry('a', 1), entry('a', 2), entry('b', 1)]); + + assert.deepEqual(output, [['a1', 'a2'], ['b1']]); + }); + + it('rejects a module whose entries arrive apart', async () => { + await assert.rejects( + regroup([ + [entry('a', 1), entry('b', 1)], + [entry('c', 1), entry('a', 2)], + ]), + /"a" arrived apart/ + ); + }); + + it('rejects a module whose entries arrive apart in one batch too', async () => { + await assert.rejects( + regroup([[entry('a', 1), entry('b', 1), entry('a', 2)]]), + /"a" arrived apart/ + ); + }); +}); + describe('getVersionFromSemVer', () => { it('returns major.x for minor 0', () => { const version = { major: 18, minor: 0, patch: 0 }; diff --git a/packages/core/src/utils/generators.mjs b/packages/core/src/utils/generators.mjs index f8996d2b5..940eac733 100644 --- a/packages/core/src/utils/generators.mjs +++ b/packages/core/src/utils/generators.mjs @@ -59,6 +59,67 @@ export const toBatches = input => { return [input]; }; +/** + * Regroups streamed entries so that each batch holds whole modules: a module + * whose entries continue in the next batch is held back and passed on with + * it. A module's entries must arrive together: one that comes back after + * another module's is an error, wherever the batches end, as its pages would + * be built without it. Entries given as one array, as a generator that streams + * its input gets them with one thread, are taken as a single batch. + * + * @template {{ api: string }} T + * @param {Array | AsyncIterable>} batches - The entries, in + * batches as they are streamed, or in one array + * @returns {AsyncGenerator>} The same entries, in batches of whole + * modules + */ +export async function* groupWholeModules(batches) { + // The modules whose entries have all arrived + const finished = new Set(); + + // The module whose entries are arriving, and those of them held back + let current; + let held = []; + + for await (const batch of toBatches(batches)) { + for (const { api } of batch) { + if (api !== current) { + if (finished.has(api)) { + throw new Error( + `The entries of "${api}" arrived apart: a module's entries must ` + + 'arrive together, in one chunk or in consecutive ones' + ); + } + + if (current !== undefined) { + finished.add(current); + } + + current = api; + } + } + + const entries = held.length > 0 ? [...held, ...batch] : batch; + + // The module the batch ends with may continue in the next one + let end = entries.length; + + while (end > 0 && entries[end - 1].api === current) { + end--; + } + + held = entries.slice(end); + + if (end > 0) { + yield entries.slice(0, end); + } + } + + if (held.length > 0) { + yield held; + } +} + /** * Parses the SemVer string into a Node.js-alike version * diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs index a2198925b..1e2768215 100644 --- a/packages/react/src/html/__tests__/generate.test.mjs +++ b/packages/react/src/html/__tests__/generate.test.mjs @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { describe, it } from 'node:test'; import { pathToFileURL } from 'node:url'; +import { collectAsyncGenerator } from '@doc-kit/core/caching.mjs'; import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; @@ -12,11 +13,29 @@ import { jsx, toJs } from 'estree-util-to-js'; import buildContent from '../../jsx-ast/utils/buildContent.mjs'; import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs'; -import { generate as chunk } from '../../section-pages/generate.mjs'; +import { generate as sectionPages } from '../../section-pages/generate.mjs'; import { compile, createViteBundler } from '../bundlers/vite.mjs'; import { FONTS } from '../constants.mjs'; import { generate } from '../generate.mjs'; +/** + * Yields the given entries as one batch, as a dependency streaming them does. + * + * @param {Array} entries + */ +async function* streamInOneBatch(entries) { + yield entries; +} + +/** + * Runs `section-pages` over the given entries, streamed in as one batch, and + * collects its pages. + * + * @param {Array} entries + */ +const chunk = entries => + collectAsyncGenerator(sectionPages(streamInOneBatch(entries))); + // Pages are rendered with the pipeline of `jsx-ast` await loadMarkdownPlugins( await loadGenerator(import.meta.resolve('../../jsx-ast/index.mjs')) diff --git a/packages/react/src/section-pages/__tests__/generate.test.mjs b/packages/react/src/section-pages/__tests__/generate.test.mjs index 8605ba1c4..48e02f6e8 100644 --- a/packages/react/src/section-pages/__tests__/generate.test.mjs +++ b/packages/react/src/section-pages/__tests__/generate.test.mjs @@ -7,6 +7,40 @@ import getConfig, { import { generate } from '../generate.mjs'; +/** + * Collects every entry the generator yields. + * + * @param {AsyncIterable>} output + * @returns {Promise>} + */ +const collect = async output => { + const entries = []; + + for await (const batch of output) { + entries.push(...batch); + } + + return entries; +}; + +/** + * Runs the generator over the given batches of entries, streamed in as + * `metadata` yields them, and collects what it yields. + * + * @param {...Array} batches + * @returns {Promise>} + */ +const run = (...batches) => collect(generate(streamOf(batches))); + +/** + * Yields each batch in turn, as a streamed input. + * + * @param {Array>} batches + */ +async function* streamOf(batches) { + yield* batches; +} + /** * Builds a metadata-shaped entry whose heading node is shared between * `heading` and `content`, exactly like the metadata generator produces. @@ -72,7 +106,7 @@ describe('section-pages generate', () => { it('keeps every original entry and appends the chunk pages', async () => { const input = createModule(); - const output = await generate(input); + const output = await run(input); assert.deepEqual(output.slice(0, input.length), input); @@ -97,7 +131,7 @@ describe('section-pages generate', () => { }); it('describes where each chunk came from', async () => { - const output = await generate(createModule()); + const output = await run(createModule()); const readFile = output.find(e => e.path === '/fs/readFile'); assert.deepEqual(readFile.chunk, { @@ -111,7 +145,7 @@ describe('section-pages generate', () => { }); it('promotes headings so the chunk reads as its own page', async () => { - const output = await generate(createModule()); + const output = await run(createModule()); const dir = output.filter(e => e.api === 'fs-Dir'); assert.deepEqual( @@ -134,13 +168,13 @@ describe('section-pages generate', () => { const input = createModule(); const snapshot = structuredClone(input); - await generate(input); + await run(input); assert.deepEqual(input, snapshot); }); it('leaves modules with a single section alone', async () => { - const output = await generate([ + const output = await run([ createEntry('tiny', 1, 'Tiny'), createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), ]); @@ -149,7 +183,7 @@ describe('section-pages generate', () => { }); it('leaves modules that document no API entry alone', async () => { - const output = await generate([ + const output = await run([ createEntry('guide', 1, 'About this documentation'), createEntry('guide', 2, 'Contributing'), createEntry('guide', 2, 'Stability index'), @@ -172,7 +206,7 @@ describe('section-pages generate', () => { link('https://example.com') ); - const output = await generate(input); + const output = await run(input); const chunk = output.find(e => e.path === '/fs/readFile'); assert.deepEqual( @@ -206,7 +240,7 @@ describe('section-pages generate', () => { }, }); - const output = await generate(input); + const output = await run(input); const chunk = output.find(e => e.path === '/fs/readFile'); assert.deepEqual( @@ -225,7 +259,9 @@ describe('section-pages generate', () => { getConfig('section-pages').exclude = ['fs']; try { - assert.equal((await generate(createModule())).length, 6); + const output = await run(createModule()); + + assert.equal(output.length, 6); } finally { getConfig('section-pages').exclude = []; } @@ -235,7 +271,7 @@ describe('section-pages generate', () => { getConfig('section-pages').maxDepth = 2; try { - const output = await generate(createModule()); + const output = await run(createModule()); assert.deepEqual( output.filter(e => e.chunk && e.heading.depth === 1).map(e => e.path), @@ -245,4 +281,73 @@ describe('section-pages generate', () => { getConfig('section-pages').maxDepth = 3; } }); + + it('builds the same output from several batches as from one', async () => { + const tiny = [ + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]; + + const whole = await run([...createModule(), ...tiny]); + const batched = await run(createModule(), tiny); + + assert.deepEqual(batched, whole); + }); + + it('builds the same output from its whole input, as it gets it with one thread', async () => { + const input = () => [ + ...createModule(), + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]; + + const whole = await collect(generate(input())); + const streamed = await run(input()); + + assert.deepEqual(whole, streamed); + }); + + it( + 'passes whole modules on before the next batch arrives', + // The next batch waits for the test, so modules held back until it arrives + // would never be passed on + { timeout: 10_000 }, + async () => { + const gate = Promise.withResolvers(); + const fs = createModule(); + + async function* batches() { + yield [...fs, createEntry('tiny', 1, 'Tiny')]; + await gate.promise; + yield [createEntry('zlib', 1, 'Zlib')]; + } + + const output = generate(batches()); + const { value } = await output.next(); + + // `fs` is whole, `tiny` may continue in the next batch + assert.deepEqual(value, fs); + + gate.resolve(); + + const rest = await collect(output); + + assert.deepEqual( + rest.filter(e => !e.chunk).map(e => e.api), + ['tiny', 'zlib'] + ); + assert.equal( + rest.filter(e => e.chunk && e.heading.depth === 1).length, + 3 + ); + } + ); + + it('builds a module split across batches as one', async () => { + const whole = await run(createModule()); + const module = createModule(); + const split = await run(module.slice(0, 2), module.slice(2)); + + assert.deepEqual(split, whole); + }); }); diff --git a/packages/react/src/section-pages/generate.mjs b/packages/react/src/section-pages/generate.mjs index 12e134cd5..eb50383f7 100644 --- a/packages/react/src/section-pages/generate.mjs +++ b/packages/react/src/section-pages/generate.mjs @@ -1,7 +1,10 @@ 'use strict'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; -import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; +import { + groupNodesByModule, + groupWholeModules, +} from '@doc-kit/core/utils/generators.mjs'; import { visit } from 'unist-util-visit'; import { headingLabel } from '../jsx-ast/utils/buildBarProps.mjs'; @@ -100,14 +103,14 @@ const buildChunkEntries = (module, chunk, index, anchors) => { }; /** - * Adds per-section chunk pages for every module, next to the full pages. + * Builds the chunk pages of every module among the given entries. * - * @type {import('./types').Generator['generate']} + * @param {Array} input - Whole modules' entries + * @param {import('./types').Configuration} config + * @returns {Array} */ -export async function generate(input) { - const { maxDepth, exclude } = getConfig('section-pages'); - - const output = [...input]; +const buildChunkPages = (input, { maxDepth, exclude }) => { + const pages = []; for (const entries of groupNodesByModule(input).values()) { const module = entries.find(entry => entry.heading.depth === 1); @@ -135,9 +138,35 @@ export async function generate(input) { const anchors = buildAnchorMap(chunks); chunks.forEach((chunk, index) => { - output.push(...buildChunkEntries(module, chunk, index, anchors)); + pages.push(...buildChunkEntries(module, chunk, index, anchors)); }); } - return output; + return pages; +}; + +/** + * Adds per-section chunk pages for every module, next to the full pages. + * + * The entries are passed on as their modules arrive, so the pages built from + * them can start while `metadata` still runs, and the chunk pages follow them + * all, in module order, as if built from the whole input at once. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input) { + const config = getConfig('section-pages'); + + const chunkPages = []; + + // A module's chunk pages need all of its entries + for await (const entries of groupWholeModules(input)) { + yield entries; + + chunkPages.push(...buildChunkPages(entries, config)); + } + + if (chunkPages.length > 0) { + yield chunkPages; + } } diff --git a/packages/react/src/section-pages/index.mjs b/packages/react/src/section-pages/index.mjs index b1043d110..380d1883c 100644 --- a/packages/react/src/section-pages/index.mjs +++ b/packages/react/src/section-pages/index.mjs @@ -29,6 +29,8 @@ export default { dependsOn: '@doc-kit/core/metadata', + streamsInput: true, + dependent: [ '@doc-kit/generator-react/html', '@doc-kit/generator-react/sitemap', diff --git a/packages/react/src/section-pages/types.d.ts b/packages/react/src/section-pages/types.d.ts index bf57e1ee2..9d6c579fc 100644 --- a/packages/react/src/section-pages/types.d.ts +++ b/packages/react/src/section-pages/types.d.ts @@ -24,5 +24,8 @@ export type Chunk = { export type Generator = GeneratorMetadata< Configuration, - Generate, Promise>> + Generate< + Array | AsyncIterable>, + AsyncGenerator> + > >;