diff --git a/.changeset/stream-metadata.md b/.changeset/stream-metadata.md new file mode 100644 index 000000000..984011c27 --- /dev/null +++ b/.changeset/stream-metadata.md @@ -0,0 +1,6 @@ +--- +'@doc-kit/core': minor +'@doc-kit/generator-react': minor +--- + +`metadata` and `section-pages` stream their input: with more than one thread, they start on their dependency's first chunk, so parsing, metadata and section pages overlap. The generated files are unchanged. `groupWholeModules`, from `@doc-kit/core/utils/generators.mjs`, regroups streamed entries so that each batch holds whole modules. diff --git a/docs/creating-generators.md b/docs/creating-generators.md index 62aa810d3..a46447404 100644 --- a/docs/creating-generators.md +++ b/docs/creating-generators.md @@ -379,7 +379,10 @@ export async function* generate(input, worker) { - With more than one thread, `input` can be iterated only once, and passing it to `worker.stream` iterates it. If you need it twice, collect it into an array first. -- The chunks are the dependency's own, in the order it yields them. +- The chunks are the dependency's own, in the order it yields them. `metadata`'s + hold whole files, but chunks need not hold whole modules: a generator that + needs all of a module's entries at once should regroup them with + `groupWholeModules` from `@doc-kit/core/utils/generators.mjs`. - Work that needs the whole input, such as sorting or writing one file, has to wait for the last chunk. - Other generators reading the same dependency are not affected: each gets diff --git a/packages/core/src/generators/metadata/index.mjs b/packages/core/src/generators/metadata/index.mjs index fcdd965b7..64747e682 100644 --- a/packages/core/src/generators/metadata/index.mjs +++ b/packages/core/src/generators/metadata/index.mjs @@ -16,6 +16,8 @@ export default { hasParallelProcessor: true, + streamsInput: true, + markdown: { remarkPlugins: [ 'remark-parse', diff --git a/packages/core/src/generators/metadata/types.d.ts b/packages/core/src/generators/metadata/types.d.ts index c84305c37..0e04b1e11 100644 --- a/packages/core/src/generators/metadata/types.d.ts +++ b/packages/core/src/generators/metadata/types.d.ts @@ -3,7 +3,10 @@ import type { Position } from 'unist'; export type Generator = GeneratorMetadata< {}, - Generate, AsyncGenerator>, + Generate< + Array | AsyncIterable>, + AsyncGenerator + >, ProcessChunk> >; diff --git a/packages/core/src/utils/__tests__/generators.test.mjs b/packages/core/src/utils/__tests__/generators.test.mjs index a9b550373..f81eb1b43 100644 --- a/packages/core/src/utils/__tests__/generators.test.mjs +++ b/packages/core/src/utils/__tests__/generators.test.mjs @@ -4,6 +4,7 @@ import { describe, it } from 'node:test'; import { getEntryDescription, groupNodesByModule, + groupWholeModules, getVersionFromSemVer, coerceSemVer, getCompatibleVersions, @@ -87,6 +88,97 @@ describe('groupNodesByModule', () => { }); }); +describe('groupWholeModules', () => { + /** + * Collects the batches regrouped from the given input, each entry as its + * module and number. + * + * @param {Array<{ api: string }> | AsyncIterable>} input + * @returns {Promise} + */ + const collect = async input => { + const output = []; + + for await (const batch of groupWholeModules(input)) { + output.push(batch.map(({ api, n }) => `${api}${n}`)); + } + + return output; + }; + + /** + * Regroups the given batches, streamed in, and collects the batches it + * yields. + * + * @param {Array>} batches + * @returns {Promise} + */ + const regroup = batches => { + async function* input() { + yield* batches; + } + + return collect(input()); + }; + + /** + * An entry of a module, told apart from the module's others by its number. + * + * @param {string} api + * @param {number} n + * @returns {{ api: string, n: number }} + */ + const entry = (api, n) => ({ api, n }); + + it('passes the entries on in the same order, each batch ending once its last module is whole', async () => { + const output = await regroup([ + [entry('a', 1), entry('a', 2), entry('b', 1)], + [entry('c', 1)], + ]); + + assert.deepEqual(output, [['a1', 'a2'], ['b1'], ['c1']]); + }); + + it('holds back a module that continues in the next batch', async () => { + const output = await regroup([ + [entry('a', 1), entry('b', 1)], + [entry('b', 2), entry('b', 3)], + [entry('b', 4), entry('c', 1)], + ]); + + assert.deepEqual(output, [['a1'], ['b1', 'b2', 'b3', 'b4'], ['c1']]); + }); + + it('skips empty batches', async () => { + const output = await regroup([[], [entry('a', 1)], []]); + + assert.deepEqual(output, [['a1']]); + }); + + it('takes entries given as one array as a single batch', async () => { + const output = await collect([entry('a', 1), entry('a', 2), entry('b', 1)]); + + assert.deepEqual(output, [['a1', 'a2'], ['b1']]); + }); + + it('rejects a module whose entries arrive apart', async () => { + await assert.rejects( + regroup([ + [entry('a', 1), entry('b', 1)], + [entry('c', 1), entry('a', 2)], + ]), + /"a" arrived apart/ + ); + }); + + it('rejects a module whose entries arrive apart in one batch too', async () => { + await assert.rejects( + regroup([[entry('a', 1), entry('b', 1), entry('a', 2)]]), + /"a" arrived apart/ + ); + }); +}); + describe('getVersionFromSemVer', () => { it('returns major.x for minor 0', () => { const version = { major: 18, minor: 0, patch: 0 }; diff --git a/packages/core/src/utils/generators.mjs b/packages/core/src/utils/generators.mjs index f8996d2b5..940eac733 100644 --- a/packages/core/src/utils/generators.mjs +++ b/packages/core/src/utils/generators.mjs @@ -59,6 +59,67 @@ export const toBatches = input => { return [input]; }; +/** + * Regroups streamed entries so that each batch holds whole modules: a module + * whose entries continue in the next batch is held back and passed on with + * it. A module's entries must arrive together: one that comes back after + * another module's is an error, wherever the batches end, as its pages would + * be built without it. Entries given as one array, as a generator that streams + * its input gets them with one thread, are taken as a single batch. + * + * @template {{ api: string }} T + * @param {Array | AsyncIterable>} batches - The entries, in + * batches as they are streamed, or in one array + * @returns {AsyncGenerator>} The same entries, in batches of whole + * modules + */ +export async function* groupWholeModules(batches) { + // The modules whose entries have all arrived + const finished = new Set(); + + // The module whose entries are arriving, and those of them held back + let current; + let held = []; + + for await (const batch of toBatches(batches)) { + for (const { api } of batch) { + if (api !== current) { + if (finished.has(api)) { + throw new Error( + `The entries of "${api}" arrived apart: a module's entries must ` + + 'arrive together, in one chunk or in consecutive ones' + ); + } + + if (current !== undefined) { + finished.add(current); + } + + current = api; + } + } + + const entries = held.length > 0 ? [...held, ...batch] : batch; + + // The module the batch ends with may continue in the next one + let end = entries.length; + + while (end > 0 && entries[end - 1].api === current) { + end--; + } + + held = entries.slice(end); + + if (end > 0) { + yield entries.slice(0, end); + } + } + + if (held.length > 0) { + yield held; + } +} + /** * Parses the SemVer string into a Node.js-alike version * diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs index a2198925b..1e2768215 100644 --- a/packages/react/src/html/__tests__/generate.test.mjs +++ b/packages/react/src/html/__tests__/generate.test.mjs @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { describe, it } from 'node:test'; import { pathToFileURL } from 'node:url'; +import { collectAsyncGenerator } from '@doc-kit/core/caching.mjs'; import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; @@ -12,11 +13,29 @@ import { jsx, toJs } from 'estree-util-to-js'; import buildContent from '../../jsx-ast/utils/buildContent.mjs'; import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs'; -import { generate as chunk } from '../../section-pages/generate.mjs'; +import { generate as sectionPages } from '../../section-pages/generate.mjs'; import { compile, createViteBundler } from '../bundlers/vite.mjs'; import { FONTS } from '../constants.mjs'; import { generate } from '../generate.mjs'; +/** + * Yields the given entries as one batch, as a dependency streaming them does. + * + * @param {Array} entries + */ +async function* streamInOneBatch(entries) { + yield entries; +} + +/** + * Runs `section-pages` over the given entries, streamed in as one batch, and + * collects its pages. + * + * @param {Array} entries + */ +const chunk = entries => + collectAsyncGenerator(sectionPages(streamInOneBatch(entries))); + // Pages are rendered with the pipeline of `jsx-ast` await loadMarkdownPlugins( await loadGenerator(import.meta.resolve('../../jsx-ast/index.mjs')) diff --git a/packages/react/src/section-pages/__tests__/generate.test.mjs b/packages/react/src/section-pages/__tests__/generate.test.mjs index 8605ba1c4..48e02f6e8 100644 --- a/packages/react/src/section-pages/__tests__/generate.test.mjs +++ b/packages/react/src/section-pages/__tests__/generate.test.mjs @@ -7,6 +7,40 @@ import getConfig, { import { generate } from '../generate.mjs'; +/** + * Collects every entry the generator yields. + * + * @param {AsyncIterable>} output + * @returns {Promise>} + */ +const collect = async output => { + const entries = []; + + for await (const batch of output) { + entries.push(...batch); + } + + return entries; +}; + +/** + * Runs the generator over the given batches of entries, streamed in as + * `metadata` yields them, and collects what it yields. + * + * @param {...Array} batches + * @returns {Promise>} + */ +const run = (...batches) => collect(generate(streamOf(batches))); + +/** + * Yields each batch in turn, as a streamed input. + * + * @param {Array>} batches + */ +async function* streamOf(batches) { + yield* batches; +} + /** * Builds a metadata-shaped entry whose heading node is shared between * `heading` and `content`, exactly like the metadata generator produces. @@ -72,7 +106,7 @@ describe('section-pages generate', () => { it('keeps every original entry and appends the chunk pages', async () => { const input = createModule(); - const output = await generate(input); + const output = await run(input); assert.deepEqual(output.slice(0, input.length), input); @@ -97,7 +131,7 @@ describe('section-pages generate', () => { }); it('describes where each chunk came from', async () => { - const output = await generate(createModule()); + const output = await run(createModule()); const readFile = output.find(e => e.path === '/fs/readFile'); assert.deepEqual(readFile.chunk, { @@ -111,7 +145,7 @@ describe('section-pages generate', () => { }); it('promotes headings so the chunk reads as its own page', async () => { - const output = await generate(createModule()); + const output = await run(createModule()); const dir = output.filter(e => e.api === 'fs-Dir'); assert.deepEqual( @@ -134,13 +168,13 @@ describe('section-pages generate', () => { const input = createModule(); const snapshot = structuredClone(input); - await generate(input); + await run(input); assert.deepEqual(input, snapshot); }); it('leaves modules with a single section alone', async () => { - const output = await generate([ + const output = await run([ createEntry('tiny', 1, 'Tiny'), createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), ]); @@ -149,7 +183,7 @@ describe('section-pages generate', () => { }); it('leaves modules that document no API entry alone', async () => { - const output = await generate([ + const output = await run([ createEntry('guide', 1, 'About this documentation'), createEntry('guide', 2, 'Contributing'), createEntry('guide', 2, 'Stability index'), @@ -172,7 +206,7 @@ describe('section-pages generate', () => { link('https://example.com') ); - const output = await generate(input); + const output = await run(input); const chunk = output.find(e => e.path === '/fs/readFile'); assert.deepEqual( @@ -206,7 +240,7 @@ describe('section-pages generate', () => { }, }); - const output = await generate(input); + const output = await run(input); const chunk = output.find(e => e.path === '/fs/readFile'); assert.deepEqual( @@ -225,7 +259,9 @@ describe('section-pages generate', () => { getConfig('section-pages').exclude = ['fs']; try { - assert.equal((await generate(createModule())).length, 6); + const output = await run(createModule()); + + assert.equal(output.length, 6); } finally { getConfig('section-pages').exclude = []; } @@ -235,7 +271,7 @@ describe('section-pages generate', () => { getConfig('section-pages').maxDepth = 2; try { - const output = await generate(createModule()); + const output = await run(createModule()); assert.deepEqual( output.filter(e => e.chunk && e.heading.depth === 1).map(e => e.path), @@ -245,4 +281,73 @@ describe('section-pages generate', () => { getConfig('section-pages').maxDepth = 3; } }); + + it('builds the same output from several batches as from one', async () => { + const tiny = [ + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]; + + const whole = await run([...createModule(), ...tiny]); + const batched = await run(createModule(), tiny); + + assert.deepEqual(batched, whole); + }); + + it('builds the same output from its whole input, as it gets it with one thread', async () => { + const input = () => [ + ...createModule(), + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]; + + const whole = await collect(generate(input())); + const streamed = await run(input()); + + assert.deepEqual(whole, streamed); + }); + + it( + 'passes whole modules on before the next batch arrives', + // The next batch waits for the test, so modules held back until it arrives + // would never be passed on + { timeout: 10_000 }, + async () => { + const gate = Promise.withResolvers(); + const fs = createModule(); + + async function* batches() { + yield [...fs, createEntry('tiny', 1, 'Tiny')]; + await gate.promise; + yield [createEntry('zlib', 1, 'Zlib')]; + } + + const output = generate(batches()); + const { value } = await output.next(); + + // `fs` is whole, `tiny` may continue in the next batch + assert.deepEqual(value, fs); + + gate.resolve(); + + const rest = await collect(output); + + assert.deepEqual( + rest.filter(e => !e.chunk).map(e => e.api), + ['tiny', 'zlib'] + ); + assert.equal( + rest.filter(e => e.chunk && e.heading.depth === 1).length, + 3 + ); + } + ); + + it('builds a module split across batches as one', async () => { + const whole = await run(createModule()); + const module = createModule(); + const split = await run(module.slice(0, 2), module.slice(2)); + + assert.deepEqual(split, whole); + }); }); diff --git a/packages/react/src/section-pages/generate.mjs b/packages/react/src/section-pages/generate.mjs index 12e134cd5..eb50383f7 100644 --- a/packages/react/src/section-pages/generate.mjs +++ b/packages/react/src/section-pages/generate.mjs @@ -1,7 +1,10 @@ 'use strict'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; -import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; +import { + groupNodesByModule, + groupWholeModules, +} from '@doc-kit/core/utils/generators.mjs'; import { visit } from 'unist-util-visit'; import { headingLabel } from '../jsx-ast/utils/buildBarProps.mjs'; @@ -100,14 +103,14 @@ const buildChunkEntries = (module, chunk, index, anchors) => { }; /** - * Adds per-section chunk pages for every module, next to the full pages. + * Builds the chunk pages of every module among the given entries. * - * @type {import('./types').Generator['generate']} + * @param {Array} input - Whole modules' entries + * @param {import('./types').Configuration} config + * @returns {Array} */ -export async function generate(input) { - const { maxDepth, exclude } = getConfig('section-pages'); - - const output = [...input]; +const buildChunkPages = (input, { maxDepth, exclude }) => { + const pages = []; for (const entries of groupNodesByModule(input).values()) { const module = entries.find(entry => entry.heading.depth === 1); @@ -135,9 +138,35 @@ export async function generate(input) { const anchors = buildAnchorMap(chunks); chunks.forEach((chunk, index) => { - output.push(...buildChunkEntries(module, chunk, index, anchors)); + pages.push(...buildChunkEntries(module, chunk, index, anchors)); }); } - return output; + return pages; +}; + +/** + * Adds per-section chunk pages for every module, next to the full pages. + * + * The entries are passed on as their modules arrive, so the pages built from + * them can start while `metadata` still runs, and the chunk pages follow them + * all, in module order, as if built from the whole input at once. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input) { + const config = getConfig('section-pages'); + + const chunkPages = []; + + // A module's chunk pages need all of its entries + for await (const entries of groupWholeModules(input)) { + yield entries; + + chunkPages.push(...buildChunkPages(entries, config)); + } + + if (chunkPages.length > 0) { + yield chunkPages; + } } diff --git a/packages/react/src/section-pages/index.mjs b/packages/react/src/section-pages/index.mjs index b1043d110..380d1883c 100644 --- a/packages/react/src/section-pages/index.mjs +++ b/packages/react/src/section-pages/index.mjs @@ -29,6 +29,8 @@ export default { dependsOn: '@doc-kit/core/metadata', + streamsInput: true, + dependent: [ '@doc-kit/generator-react/html', '@doc-kit/generator-react/sitemap', diff --git a/packages/react/src/section-pages/types.d.ts b/packages/react/src/section-pages/types.d.ts index bf57e1ee2..9d6c579fc 100644 --- a/packages/react/src/section-pages/types.d.ts +++ b/packages/react/src/section-pages/types.d.ts @@ -24,5 +24,8 @@ export type Chunk = { export type Generator = GeneratorMetadata< Configuration, - Generate, Promise>> + Generate< + Array | AsyncIterable>, + AsyncGenerator> + > >;