From 65d6aeb4855f743ad592d2e29c03763a83fc2ac7 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Tue, 3 Nov 2026 19:04:49 +0100 Subject: [PATCH] perf: let jsx-ast and html start on their dependency's first chunk With more than one thread, jsx-ast builds pages as their entries arrive and html compiles, bundles and renders them as it receives them, so page building and rendering overlap with the generators before them. The generated files are unchanged. jsx-ast yields its pages in the order their entries arrived, and once every page has, a { manifest } item with all their heads in that order. html compiles each page as it comes, bundles the component library and the client assets as soon as the manifest does, and has the workers render the pages as they are compiled. To list the pages as the site does, headingSortFn is now exported. Bundler adapters' compile can now run before and alongside buildServer and buildClient. createChannel is the queue html hands its compiled pages to the workers on, and merge yields from several async iterables as items arrive: jsx-ast announces the manifest next to its pages with it. Assisted-by: Claude Opus 5.5 Assisted-by: Claude Sonnet 5.5 --- .changeset/stream-pages.md | 6 + .../core/src/utils/__tests__/channel.test.mjs | 63 ++++ .../src/utils/__tests__/generators.test.mjs | 54 +++ packages/core/src/utils/channel.mjs | 75 ++++ packages/core/src/utils/generators.mjs | 49 +++ packages/react/package.json | 1 + packages/react/src/html/README.md | 34 +- .../src/html/__tests__/generate.test.mjs | 328 +++++++++++++++++- packages/react/src/html/generate.mjs | 270 ++++++++++---- packages/react/src/html/index.mjs | 13 +- packages/react/src/html/types.d.ts | 7 +- packages/react/src/html/utils/all.mjs | 15 +- packages/react/src/html/utils/generate.mjs | 2 +- packages/react/src/html/utils/render.mjs | 33 +- packages/react/src/jsx-ast/README.md | 12 + .../src/jsx-ast/__tests__/generate.test.mjs | 232 ++++++++++++- packages/react/src/jsx-ast/generate.mjs | 73 +++- packages/react/src/jsx-ast/index.mjs | 2 + packages/react/src/jsx-ast/types.d.ts | 5 +- .../src/jsx-ast/utils/getSortedHeadNodes.mjs | 5 +- 20 files changed, 1145 insertions(+), 134 deletions(-) create mode 100644 .changeset/stream-pages.md create mode 100644 packages/core/src/utils/__tests__/channel.test.mjs create mode 100644 packages/core/src/utils/channel.mjs diff --git a/.changeset/stream-pages.md b/.changeset/stream-pages.md new file mode 100644 index 000000000..3e04b646c --- /dev/null +++ b/.changeset/stream-pages.md @@ -0,0 +1,6 @@ +--- +'@doc-kit/core': minor +'@doc-kit/generator-react': minor +--- + +`jsx-ast` and `html` stream their input: with more than one thread, page building and rendering overlap with the generators before them. `html` bundles as soon as every page's metadata is known, and renders pages as they are built. Streaming leaves the generated files unchanged, but `jsx-ast` now yields pages in the order their entries arrive, not the order the site lists them, and, once every page has arrived, a `{ manifest }` item listing their heads in that order. To list the pages as the site does, skip the `{ manifest }` item and sort them with `headingSortFn`, now exported from `@doc-kit/generator-react/jsx-ast/utils/getSortedHeadNodes`, as `pages.toSorted((a, b) => headingSortFn(a.data, b.data))`, keeping the synthetic pages (`data.synthetic`) last. Bundler adapters' `compile` can now be called before and while `buildServer` and `buildClient` run. `createChannel` and `merge`, from `@doc-kit/core/utils`, are the queue and the merge of async iterables they are built on. diff --git a/packages/core/src/utils/__tests__/channel.test.mjs b/packages/core/src/utils/__tests__/channel.test.mjs new file mode 100644 index 000000000..7f5a9d4c8 --- /dev/null +++ b/packages/core/src/utils/__tests__/channel.test.mjs @@ -0,0 +1,63 @@ +import { deepStrictEqual } from 'node:assert'; +import { describe, it } from 'node:test'; + +import { createChannel } from '../channel.mjs'; + +/** + * Reads every batch of an async iterable. + * + * @template T + * @param {AsyncIterable} iterable + * @returns {Promise} + */ +async function readBatches(iterable) { + const batches = []; + + for await (const batch of iterable) { + batches.push(batch); + } + + return batches; +} + +describe('createChannel', () => { + it('gives the batches as they are added, until it ends', async () => { + const channel = createChannel(); + const reading = readBatches(channel); + + channel.push([1]); + await new Promise(resolve => setImmediate(resolve)); + channel.push([2]); + channel.end(); + + const batches = await reading; + + deepStrictEqual(batches, [[1], [2]]); + }); + + it('gives a prepended batch before those not read yet', async () => { + const channel = createChannel(); + + channel.push([1]); + channel.push([2]); + channel.prepend([0]); + channel.end(); + + const batches = await readBatches(channel); + + deepStrictEqual(batches, [[0], [1], [2]]); + }); + + it('drops the batches not read yet when cleared', async () => { + const channel = createChannel(); + + channel.push([1]); + channel.clear(); + channel.push([2]); + channel.end(); + + const batches = await readBatches(channel); + + deepStrictEqual(batches, [[2]]); + }); +}); diff --git a/packages/core/src/utils/__tests__/generators.test.mjs b/packages/core/src/utils/__tests__/generators.test.mjs index f81eb1b43..210f6bcce 100644 --- a/packages/core/src/utils/__tests__/generators.test.mjs +++ b/packages/core/src/utils/__tests__/generators.test.mjs @@ -8,6 +8,7 @@ import { getVersionFromSemVer, coerceSemVer, getCompatibleVersions, + merge, toBatches, } from '../generators.mjs'; @@ -244,3 +245,56 @@ describe('toBatches', () => { assert.equal(toBatches(streamed), streamed); }); }); + +describe('merge', () => { + it('yields each item as soon as it arrives, until every source ends', async () => { + const gate = Promise.withResolvers(); + + async function* slow() { + await gate.promise; + + yield 'slow'; + } + + async function* fast() { + yield 'fast'; + yield 'faster'; + } + + const items = []; + + for await (const item of merge(slow(), fast())) { + items.push(item); + + if (item === 'faster') { + gate.resolve(); + } + } + + assert.deepEqual(items, ['fast', 'faster', 'slow']); + }); + + it('throws a failure without waiting for the other sources', async () => { + async function* never() { + await new Promise(() => {}); + + yield 'never'; + } + + async function* failing() { + yield 'before'; + + throw new Error('merged failure'); + } + + const items = []; + + await assert.rejects(async () => { + for await (const item of merge(never(), failing())) { + items.push(item); + } + }, /merged failure/); + + assert.deepEqual(items, ['before']); + }); +}); diff --git a/packages/core/src/utils/channel.mjs b/packages/core/src/utils/channel.mjs new file mode 100644 index 000000000..1aaeaa355 --- /dev/null +++ b/packages/core/src/utils/channel.mjs @@ -0,0 +1,75 @@ +'use strict'; + +import { createSignal } from './misc.mjs'; + +/** + * Creates a queue of batches that can be read as an async iterable while it is + * still being filled. + * + * @template T + */ +export const createChannel = () => { + /** @type {Array} */ + const batches = []; + + // Notified as batches are added, and at the end + const added = createSignal(); + + let ended = false; + + return { + /** + * Adds a batch, after those not read yet. + * + * @param {T[]} batch + */ + push(batch) { + batches.push(batch); + added.notify(); + }, + + /** + * Adds a batch, before those not read yet. + * + * @param {T[]} batch + */ + prepend(batch) { + batches.unshift(batch); + added.notify(); + }, + + /** + * Drops the batches not read yet. + */ + clear() { + batches.length = 0; + }, + + /** + * Ends the channel: the reader stops once it has read every batch. + */ + end() { + ended = true; + added.notify(); + }, + + /** + * Reads the batches as they are added, until the channel ends. + * + * @returns {AsyncGenerator} + */ + async *[Symbol.asyncIterator]() { + for (;;) { + while (batches.length === 0) { + if (ended) { + return; + } + + await added.wait(); + } + + yield batches.shift(); + } + }, + }; +}; diff --git a/packages/core/src/utils/generators.mjs b/packages/core/src/utils/generators.mjs index 940eac733..d64fdb7d4 100644 --- a/packages/core/src/utils/generators.mjs +++ b/packages/core/src/utils/generators.mjs @@ -2,6 +2,7 @@ import { coerce, major } from 'semver'; +import { createChannel } from './channel.mjs'; import { isAsyncIterable } from './misc.mjs'; import { transformNodeToString } from './unist.mjs'; @@ -59,6 +60,54 @@ export const toBatches = input => { return [input]; }; +/** + * Reads several async iterables at once, yielding each item as soon as it + * arrives, until they have all ended. A failure is thrown once the items that + * arrived before it are yielded, without waiting for the other iterables. + * + * @template T + * @param {...AsyncIterable} sources + * @returns {AsyncGenerator} + */ +export async function* merge(...sources) { + const channel = createChannel(); + + let pending = sources.length; + let failure; + + /** + * Passes a source's items on as they arrive, and ends the channel once every + * source has ended, or one fails. + * + * @param {AsyncIterable} source + */ + const pipe = async source => { + try { + for await (const item of source) { + channel.push(item); + } + } catch (error) { + failure ??= { error }; + } finally { + pending--; + + if (failure || pending === 0) { + channel.end(); + } + } + }; + + for (const source of sources) { + pipe(source); + } + + yield* channel; + + if (failure) { + throw failure.error; + } +} + /** * Regroups streamed entries so that each batch holds whole modules: a module * whose entries continue in the next batch is held back and passed on with diff --git a/packages/react/package.json b/packages/react/package.json index ed0005777..c69d2c9bd 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -15,6 +15,7 @@ "./html/ui/*": "./src/html/ui/*", "./html/bundlers/vite": "./src/html/bundlers/vite.mjs", "./jsx-ast": "./src/jsx-ast/index.mjs", + "./jsx-ast/utils/getSortedHeadNodes": "./src/jsx-ast/utils/getSortedHeadNodes.mjs", "./llms-txt": "./src/llms-txt/index.mjs", "./llms-txt-full": "./src/llms-txt-full/index.mjs", "./orama-db": "./src/orama-db/index.mjs", diff --git a/packages/react/src/html/README.md b/packages/react/src/html/README.md index e045a5f46..a929869c4 100644 --- a/packages/react/src/html/README.md +++ b/packages/react/src/html/README.md @@ -6,12 +6,15 @@ assets, written to `output`. Vite is the default bundler adapter, but projects can supply an adapter for webpack or another bundler. The generator is output-only and does not return an in-memory copy of its HTML or CSS. -The site is built in pieces that are each as small as they can be. The bundler -builds the component library once and the client assets once. Each page's -program is then compiled — JSX to a plain module — and the worker pool imports, -renders, templates, minifies and writes the pages one at a time, so memory -scales with the largest page rather than with the site. `all.html` is assembled -from the module pages' compiled content rather than built again from scratch. +The site is built in pieces that are each as small as they can be, each as +soon as what it needs is there. Each page's program is compiled — JSX to a +plain module — as `jsx-ast` builds the page. The bundler builds the component +library once and the client assets once, as soon as every page's metadata is +known, and the worker pool imports, renders, templates, minifies and writes +the pages one at a time as they are compiled, so memory scales with the largest +page rather than with the site. With one thread, each step waits for the one +before it to finish. `all.html` is assembled from the module pages' compiled +content rather than built again from scratch. ## Configuring @@ -201,16 +204,21 @@ for a particular build system. `buildServer` receives `{ entry, virtualImports, outDir, config }`. The `entry` is the component library's source: re-exports of every component a page may render, Preact's `h` and `Fragment`, and `renderToStringAsync`. It must be -bundled into one self-contained module written under `outDir` (a temporary -directory the generator removes afterwards), since the page programs import it -from wherever they are compiled to. +bundled into one self-contained module written under `outDir` (an empty +temporary directory the generator removes afterwards), since the page programs +import it from wherever they are compiled to. `compile(code, fileName)` receives one page program: a module that imports the library and exports the page's `content` and a default function rendering the -page from its layout props, written in JSX. It must return plain JavaScript. The JSX must compile with the classic -runtime to calls of the `_jsx` and `_Fragment` bindings the program imports -(these names are exported as `JSX_PRAGMA` and `JSX_PRAGMA_FRAG` from the -generator's `constants.mjs`), so that the page and the library share one Preact. +page from its layout props, written in JSX. It must return plain JavaScript. +Pages are compiled as they are built, so `compile` can be called before +`buildServer`, and while `buildServer` or `buildClient` still runs: it must +leave the program's imports as they are, since the library they point to may +not exist yet, and must not depend on the builds. The JSX must compile with the +classic runtime to calls of the `_jsx` and `_Fragment` bindings the program +imports (these names are exported as `JSX_PRAGMA` and `JSX_PRAGMA_FRAG` from +the generator's `constants.mjs`), so that the page and the library share one +Preact. `buildClient` receives `{ entry, virtualImports, config }`. The client `entry` is a single program shared by every page. It must be bundled into diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs index 1e2768215..e0e8533b3 100644 --- a/packages/react/src/html/__tests__/generate.test.mjs +++ b/packages/react/src/html/__tests__/generate.test.mjs @@ -3,6 +3,7 @@ import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it } from 'node:test'; +import { setTimeout } from 'node:timers/promises'; import { pathToFileURL } from 'node:url'; import { collectAsyncGenerator } from '@doc-kit/core/caching.mjs'; @@ -16,17 +17,26 @@ import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs'; import { generate as sectionPages } from '../../section-pages/generate.mjs'; import { compile, createViteBundler } from '../bundlers/vite.mjs'; import { FONTS } from '../constants.mjs'; -import { generate } from '../generate.mjs'; +import { generate as generatePages } from '../generate.mjs'; +import { processChunk } from '../utils/render.mjs'; /** - * Yields the given entries as one batch, as a dependency streaming them does. + * Yields the given items as one batch, as a dependency streaming them does. * - * @param {Array} entries + * @param {Array} items */ -async function* streamInOneBatch(entries) { - yield entries; +async function* streamInOneBatch(items) { + yield items; } +/** + * Runs the generator over the given pages, streamed in as one batch, as + * `jsx-ast` yields them. + * + * @param {Array} pages + */ +const generate = pages => generatePages(streamInOneBatch(pages)); + /** * Runs `section-pages` over the given entries, streamed in as one batch, and * collects its pages. @@ -101,6 +111,101 @@ const createTestConfiguration = async (context, target = ['html']) => { return { config, output }; }; +/** + * A page as `jsx-ast` emits it, its content a plain paragraph. + * + * @param {string} api + * @param {string} text + */ +const createPage = (api, text) => ({ + data: createEntry(api, text), + headings: [{ depth: 1, value: text, slug: api }], + readingTime: undefined, + content: `<>

${text} body

`, +}); + +/** + * A stand-in for the component library: the JSX runtime, and a renderer that + * writes the elements out as plain markup. + */ +const STAND_IN_LIBRARY = [ + 'export const h = (type, props, ...children) => ({ type, props: { ...props, children } });', + 'export const Fragment = ({ children }) => children;', + 'export const Layout = ({ children }) => children;', + 'const render = node =>', + ' Array.isArray(node) ? node.map(render).join("")', + ' : typeof node?.type === "function" ? render(node.type(node.props))', + ' : typeof node?.type === "string" ? `<${node.type}>${render(node.props.children)}`', + ' : String(node ?? "");', + 'export const renderToStringAsync = async node => render(node);', +].join('\n'); + +/** + * A stand-in bundler: it builds the stand-in library, compiles the pages + * with the real compiler, and has no client assets. It records each program + * it compiles, by file name, and each bundle it builds. + * + * @param {Array} calls + * @param {{ bundled?: PromiseWithResolvers, slow?: boolean }} [options] + * `bundled` is resolved once both bundles are built; `slow` makes the + * library take a while, as a real one does, so that a generator that didn't + * wait for it would remove its directory while it is being built. + */ +const createStandInBundler = (calls, { bundled, slow = false } = {}) => ({ + async buildServer({ outDir }) { + calls.push('server'); + + if (slow) { + await setTimeout(50); + } + + const library = join(outDir, 'library.mjs'); + + await writeFile(library, STAND_IN_LIBRARY); + + return pathToFileURL(library).href; + }, + + compile(code, fileName) { + calls.push(fileName); + + return compile(code, fileName); + }, + + async buildClient() { + calls.push('client'); + bundled?.resolve(); + + return { scripts: [], preloads: [], stylesheets: [] }; + }, +}); + +/** + * The temporary directories the generator builds the site in. + */ +const listBuildDirectories = async () => { + const names = await readdir(tmpdir()); + + return names.filter(name => name.startsWith('doc-kit-html-')); +}; + +/** + * Every page written to the output directory, by file name. + * + * @param {string} output + */ +const readPages = async output => { + const names = await readdir(output, { recursive: true }); + const pages = names.filter(name => name.endsWith('.html')).sort(); + const contents = await Promise.all( + pages.map(name => readFile(join(output, name), 'utf8')) + ); + + return Object.fromEntries( + pages.map((name, index) => [name, contents[index]]) + ); +}; + describe('web generate', () => { it('writes bundled HTML and omits View As links for synthetic pages', async context => { const { config, output } = await createTestConfiguration(context); @@ -399,6 +504,217 @@ describe('web generate', () => { html, // ); - assert.deepStrictEqual(calls, ['server', 'client', 'compile']); + // The page is compiled as it arrives, before the library is built + assert.deepStrictEqual(calls, ['compile', 'server', 'client']); + }); + + describe('with the manifest', () => { + const zlib = createPage('zlib', 'Zlib'); + const fs = createPage('fs', 'File system'); + const index = createPage('index', 'Index'); + const manifest = [zlib, fs, index].map(({ data }) => data); + + it( + 'bundles once the manifest arrives, and compiles all.html once its parts are', + // The input waits for the bundles, so bundles that wait for the whole + // input would never be built + { timeout: 10_000 }, + async context => { + const { config, output } = await createTestConfiguration(context); + const calls = []; + const bundled = Promise.withResolvers(); + + config.threads = 2; + config.html.generateAllPage = true; + config.html.bundler = createStandInBundler(calls, { bundled }); + + async function* input() { + yield [zlib]; + yield [{ manifest }]; + + // The bundles are built without waiting for the pages still to come + await bundled.promise; + + yield [fs]; + yield [index]; + } + + await generatePages(input()); + + // all.html is compiled once, as soon as its last part has been + assert.deepStrictEqual(calls, [ + 'zlib.jsx', + 'server', + 'client', + 'fs.jsx', + 'all.jsx', + 'index.jsx', + ]); + + const html = await readFile(join(output, 'all.html'), 'utf8'); + + // In sidebar order, the index left out + assert.match(html, /File system body[\s\S]*Zlib body/); + assert.doesNotMatch(html, /Index body/); + } + ); + + it('builds the same site from its whole input, as it gets it with one thread', async context => { + /** + * Builds the site from the given input and reads its pages. + * + * @param {number} threads + * @param {Array | AsyncIterable>} input + */ + const build = async (threads, input) => { + const { config, output } = await createTestConfiguration(context); + + config.threads = threads; + config.html.generateAllPage = true; + config.html.bundler = createStandInBundler([]); + + await generatePages(input); + + return readPages(output); + }; + + async function* batches() { + yield [zlib]; + yield [{ manifest }]; + yield [fs]; + yield [index]; + } + + const streamed = await build(2, batches()); + const whole = await build(1, [zlib, fs, { manifest }, index]); + + assert.deepStrictEqual(Object.keys(whole), [ + 'all.html', + 'fs.html', + 'index.html', + 'zlib.html', + ]); + assert.deepStrictEqual(whole, streamed); + }); + + it( + 'rejects when its input fails before the manifest, leaving no temporary files', + { timeout: 10_000 }, + async context => { + const { config, output } = await createTestConfiguration(context); + const before = await listBuildDirectories(); + + config.threads = 2; + config.html.bundler = createStandInBundler([]); + + async function* input() { + yield [zlib]; + + throw new Error('jsx-ast failed'); + } + + await assert.rejects(generatePages(input()), /jsx-ast failed/); + + const after = await listBuildDirectories(); + const pages = await readPages(output); + + assert.deepStrictEqual( + after.filter(name => !before.includes(name)), + [] + ); + assert.deepStrictEqual(pages, {}); + } + ); + + it( + 'writes no page once its input has failed after the manifest, and leaves no temporary files', + { timeout: 10_000 }, + async context => { + const { config, output } = await createTestConfiguration(context); + const before = await listBuildDirectories(); + const calls = []; + + config.threads = 2; + // Still bundling when the input fails + config.html.bundler = createStandInBundler(calls, { slow: true }); + + async function* input() { + yield [zlib]; + yield [{ manifest }]; + + throw new Error('jsx-ast failed'); + } + + await assert.rejects(generatePages(input()), /jsx-ast failed/); + + const after = await listBuildDirectories(); + const pages = await readPages(output); + + // It waited for the bundles before removing the directory they are + // written to + assert.deepStrictEqual(calls, ['zlib.jsx', 'server', 'client']); + assert.deepStrictEqual( + after.filter(name => !before.includes(name)), + [] + ); + assert.deepStrictEqual(pages, {}); + } + ); + + it( + 'writes none of the pages still waiting for a worker once its input has failed', + // The worker waits for the input to fail + { timeout: 10_000 }, + async context => { + const { config, output } = await createTestConfiguration(context); + const rendering = Promise.withResolvers(); + const failed = Promise.withResolvers(); + + config.threads = 2; + config.html.bundler = createStandInBundler([]); + + // Renders on this thread, a batch at a time, as the generator does + // without a pool, but the first only once the input has failed + const worker = { + async *stream(batches, extra) { + for await (const tasks of batches) { + rendering.resolve(); + + await failed.promise; + + const written = await processChunk( + tasks, + [...tasks.keys()], + extra + ); + + yield written; + } + }, + }; + + async function* input() { + yield [zlib]; + yield [{ manifest }]; + + // While the zlib page is being rendered, so this one waits for it + await rendering.promise; + + yield [fs]; + + // The failure reaches the generator within this turn of the event + // loop + setImmediate(failed.resolve); + + throw new Error('jsx-ast failed'); + } + + await assert.rejects(generatePages(input(), worker), /jsx-ast failed/); + + const pages = await readPages(output); + + assert.deepStrictEqual(Object.keys(pages), ['zlib.html']); + } + ); }); }); diff --git a/packages/react/src/html/generate.mjs b/packages/react/src/html/generate.mjs index 83f7ba23e..85cdef2d8 100644 --- a/packages/react/src/html/generate.mjs +++ b/packages/react/src/html/generate.mjs @@ -1,15 +1,19 @@ 'use strict'; +import { rmSync } from 'node:fs'; import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { pathToFileURL } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import logger from '@doc-kit/core/logger/index.mjs'; +import { createChannel } from '@doc-kit/core/utils/channel.mjs'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; +import { toBatches } from '@doc-kit/core/utils/generators.mjs'; +import { headingSortFn } from '../jsx-ast/utils/getSortedHeadNodes.mjs'; import { resolveBundler } from './bundlers/index.mjs'; -import { buildAllPage } from './utils/all.mjs'; +import { buildAllPage, createAllPageHead, isModulePage } from './utils/all.mjs'; import { copyStaticAssets } from './utils/copying.mjs'; import createProgramBuilder, { moduleFileName } from './utils/generate.mjs'; import { createPageWriter } from './utils/render.mjs'; @@ -19,18 +23,13 @@ const htmlLogger = logger.child('html'); /** * Main generation function: turns the pages' JSX into the static site. * - * Receives `jsx-ast`'s output as `{ data, headings, readingTime, content }` - * items, `content` being each page's JSX code. The site is then built in - * pieces that are each as small as they can be: - * - * 1. The component library is bundled once, for the server. - * 2. The client assets are bundled once; every page loads the same ones. - * 3. Each page's program is compiled (JSX to a plain module) and written to a - * temporary directory, one at a time, so no page is held longer than that. - * 4. `all.html`, when enabled, is a program that imports the module pages' - * content, so it is compiled from what was already compiled. - * 5. The worker pool imports, renders, templates, minifies and writes the - * pages, one page in memory per worker. + * Receives `jsx-ast`'s pages as it builds them, `content` being each page's JSX + * code, and a `{ manifest }` of every page's head once it knows them all. Each + * page's program is compiled to a plain module as it arrives, the component + * library and the client assets are bundled once every page's metadata is + * known, and the worker pool renders the pages as they are compiled, one in + * memory per worker. `all.html` imports the module pages' content, so it is + * compiled once they all are. With one thread, the input arrives whole. * * @type {import('./types').Generator['generate']} */ @@ -43,18 +42,23 @@ export async function generate(input, worker) { // page is written; the directory is removed afterwards const outDir = await mkdtemp(join(tmpdir(), 'doc-kit-html-')); + /** + * Removes the directory if another generator's failure ends the process + * before this one returns. + */ + const removeOnExit = () => rmSync(outDir, { recursive: true, force: true }); + + process.once('exit', removeOnExit); + try { - const { assets, tasks } = await buildPrograms({ - pages: [...input], - outDir, - config, - }); + const building = buildSite(input, { worker, outDir, config, template }); - // The pages' programs are compiled, so rendering doesn't need their code input = undefined; - await createPageWriter(worker)(tasks, { template, assets }); + await building; } finally { + process.off('exit', removeOnExit); + await rm(outDir, { recursive: true, force: true }); } @@ -62,24 +66,23 @@ export async function generate(input, worker) { } /** - * Bundles the component library and the client assets, and compiles every - * page's program into `outDir`, for the worker pool to render. - * - * Everything it sets up for that, the bundler included, is let go once it - * returns: with one thread, the pages are rendered on this thread next. + * Compiles every page's program into `outDir` as the pages arrive, bundles the + * component library and the client assets as soon as every page's metadata is + * known, and has the worker pool render the pages as they are compiled. * + * @param {Parameters[0]} input - The pages and the manifest, as they arrive, or all at once with one thread * @param {object} options - * @param {Array} options.pages - Every page, in render order + * @param {ParallelWorker} [options.worker] - Renders the pages; without it, they are rendered on this thread * @param {string} options.outDir - Where the library and the programs are written * @param {import('./types').ResolvedWebConfiguration} options.config - * @returns {Promise<{ assets: import('./types').ClientAssets, tasks: Array }>} + * @param {string} options.template - The page template + * @returns {Promise} */ -const buildPrograms = async ({ pages, outDir, config }) => { - const all = config.generateAllPage ? buildAllPage(pages) : undefined; - - // Every page's metadata, in render order — the sidebar, the index and the - // cross links need the whole set. - const datas = [...pages, ...(all ? [all] : [])].map(({ data }) => data); +const buildSite = async (input, options) => { + // Destructured here rather than in the parameters: with a destructuring + // parameter, the arguments keep their own copy of `input`, which letting go + // of `input` below would not let go of + const { worker, outDir, config, template } = options; // Loaded here rather than with the generator, so the threads rendering // pages, which load this module too, never load what only this needs @@ -89,26 +92,13 @@ const buildPrograms = async ({ pages, outDir, config }) => { const { buildLibraryProgram, buildPageProgram, clientProgram } = createProgramBuilder(); - const libraryURL = await bundler.buildServer({ - entry: buildLibraryProgram(), - virtualImports: createVirtualImports(datas, config.virtualImports, true), - outDir, - config, - }); - - htmlLogger.debug('Built the component library'); - - const assets = await bundler.buildClient({ - entry: clientProgram, - virtualImports: createVirtualImports(datas, config.virtualImports, false), - config, - }); - - htmlLogger.debug('Built the client assets', assets); - const modulesDir = join(outDir, 'pages'); await mkdir(modulesDir); + // Pages are compiled before the library is built, so they import it through + // this module, which re-exports it once it is + const libraryURL = pathToFileURL(join(outDir, 'page-library.mjs')).href; + /** * Compiles a page's program to disk and describes it for the workers. * @@ -136,25 +126,171 @@ const buildPrograms = async ({ pages, outDir, config }) => { }; }; - const tasks = []; + /** + * Bundles the component library and the client assets. + * + * @param {Array} heads - Every page's metadata, in the order the pages arrived + * @returns {Promise} + */ + const bundle = async heads => { + // Every page's metadata, in the order the site lists the pages — the + // sidebar, the index and the cross links need the whole set + const datas = [ + ...heads.filter(data => !data.synthetic).toSorted(headingSortFn), + ...heads.filter(data => data.synthetic), + ...(config.generateAllPage ? [createAllPageHead()] : []), + ]; - for (const page of pages) { - tasks.push(await compile(page)); - } + // An empty directory of its own: the compiled pages are already in `outDir` + const libraryDir = join(outDir, 'library'); + await mkdir(libraryDir); - // The composed page imports the other pages' compiled programs, which - // exist from here on, so it is rendered by the worker pool alongside them. - // Rendering it on this thread instead would hold the whole site in it and - // block the pool from shutting down its idle workers until it is done. - // It goes first: it takes by far the longest, so the other pages are - // rendered while it is. - if (all) { - const allTask = await compile(all); + const builtLibraryURL = await bundler.buildServer({ + entry: buildLibraryProgram(), + virtualImports: createVirtualImports(datas, config.virtualImports, true), + outDir: libraryDir, + config, + }); - tasks.unshift(allTask); - } + await writeFile( + fileURLToPath(libraryURL), + `export * from ${JSON.stringify(builtLibraryURL)};\n` + ); + + htmlLogger.debug('Built the component library'); - htmlLogger.debug(`Compiled ${tasks.length} page programs`); + const assets = await bundler.buildClient({ + entry: clientProgram, + virtualImports: createVirtualImports(datas, config.virtualImports, false), + config, + }); + + htmlLogger.debug('Built the client assets', assets); + + return assets; + }; + + // The compiled pages, rendered once the bundles are built + const rendering = createChannel(); + + let writing; + + /** + * Starts bundling, then rendering the compiled pages. + * + * @param {Array} heads - Every page's metadata + */ + const startWriting = heads => { + writing = bundle(heads).then(assets => + createPageWriter(worker)(rendering, { template, assets }) + ); + + // Thrown once every compiled page has been handed over + writing.catch(() => {}); + }; + + const compiled = []; + + let allWritten = !config.generateAllPage; + + // The module pages `all.html` is made of that aren't compiled yet + let missingParts; + + /** + * Compiles `all.html`, which imports the module pages' compiled programs, + * and has the worker pool render it first, as it takes by far the longest. + */ + const writeAllPage = async () => { + const allTask = await compile(buildAllPage(compiled)); + + rendering.prepend([allTask]); + + allWritten = true; + }; + + // With one thread, pages are rendered on this thread once all are compiled, + // so the manifest is kept until then + let manifest; + + const alongside = getConfig().threads > 1; + + /** + * Compiles each page as it arrives, and hands it over. A function of its + * own, so that once it returns nothing holds the pages' code. + * + * @param {typeof input} pages + */ + const compilePages = async pages => { + for await (const items of toBatches(pages)) { + const tasks = []; + + for (const item of items) { + if ('manifest' in item) { + if (alongside) { + startWriting(item.manifest); + } else { + manifest = item.manifest; + } + + missingParts = new Set( + item.manifest.filter(isModulePage).map(({ api }) => api) + ); + + for (const { data } of compiled) { + missingParts.delete(data.api); + } + + continue; + } + + const task = await compile(item); + + compiled.push(task); + tasks.push(task); + + missingParts?.delete(task.data.api); + } + + if (tasks.length > 0) { + rendering.push(tasks); + } + + if (!allWritten && missingParts?.size === 0) { + await writeAllPage(); + } + } + }; + + try { + const compiling = compilePages(input); + + input = undefined; + + await compiling; + + // With one thread, or without a manifest + if (!writing) { + startWriting(manifest ?? compiled.map(({ data }) => data)); + + manifest = undefined; + } + + if (!allWritten) { + await writeAllPage(); + } + + htmlLogger.debug(`Compiled ${compiled.length} page programs`); + } catch (error) { + // Pages still waiting for a worker aren't written + rendering.clear(); + + throw error; + } finally { + rendering.end(); + + // The bundler and the workers may still use `outDir`, removed after this + await writing?.catch(() => {}); + } - return { assets, tasks }; + await writing; }; diff --git a/packages/react/src/html/index.mjs b/packages/react/src/html/index.mjs index 607592a39..b65c912aa 100644 --- a/packages/react/src/html/index.mjs +++ b/packages/react/src/html/index.mjs @@ -13,11 +13,12 @@ import { processChunk } from './utils/render.mjs'; * - Client-side JavaScript with code splitting * - Bundled CSS styles * - * The configured bundler builds the component library and the client assets - * once each; the pages are then compiled, rendered, templated, minified and - * written one at a time by the worker pool, so memory scales with the largest - * page rather than with the site. `all.html` is assembled from the module - * pages' compiled content instead of being built again from scratch. + * Each page is compiled as `jsx-ast` builds it. Once every page's metadata is + * known, the configured bundler builds the component library and the client + * assets once each, and the worker pool renders, templates, minifies and writes + * the pages one at a time as they are compiled, so memory scales with the + * largest page rather than with the site. `all.html` is assembled from the + * module pages' compiled content instead of being built again from scratch. * * This terminal generator writes to the output directory and does not return * an in-memory copy. @@ -31,6 +32,8 @@ export default { dependsOn: '@doc-kit/generator-react/jsx-ast', + streamsInput: true, + /** * @param {import('@doc-kit/core/utils/configuration/types').Configuration} config */ diff --git a/packages/react/src/html/types.d.ts b/packages/react/src/html/types.d.ts index 5bfb72510..73bbced67 100644 --- a/packages/react/src/html/types.d.ts +++ b/packages/react/src/html/types.d.ts @@ -1,3 +1,4 @@ +import type { MetadataEntry } from '@doc-kit/core/generators/metadata/types'; import type { GlobalConfiguration } from '@doc-kit/core/utils/configuration/types'; import type NavBar from '@node-core/ui-components/Containers/NavBar'; import type SideBar from '@node-core/ui-components/Containers/Sidebar'; @@ -122,6 +123,10 @@ export type Configuration = { export type Generator = GeneratorMetadata< Configuration, - Generate, Promise>, + Generate< + | Array }> + | AsyncIterable }>>, + Promise + >, ProcessChunk >; diff --git a/packages/react/src/html/utils/all.mjs b/packages/react/src/html/utils/all.mjs index 0c254f0f3..108357483 100644 --- a/packages/react/src/html/utils/all.mjs +++ b/packages/react/src/html/utils/all.mjs @@ -10,18 +10,25 @@ import { createSyntheticHead } from '../../jsx-ast/utils/synthetic/synthetic.mjs * * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} data */ -const isModulePage = data => +export const isModulePage = data => !data.synthetic && !data.chunk && data.api !== 'index'; /** - * Builds the `all.html` page from the module pages already generated. + * The metadata of `all.html`, which the bundles need before its parts are built. + * + * @returns {import('@doc-kit/core/generators/metadata/types').MetadataEntry} + */ +export const createAllPageHead = () => createSyntheticHead('all', 'All'); + +/** + * Builds the `all.html` page from the module pages already compiled. * * Its content is exactly the module pages' content, one after the other, so * nothing is rebuilt: the page program imports each module's `content` export * (see `buildPageProgram`) and the table of contents is the modules' own, * concatenated. Modules are ordered as the sidebar lists them. * - * @param {Array} pages - Every generated page + * @param {Array>} pages - The pages compiled, every module page among them * @returns {import('../types').ComposedPage} */ export const buildAllPage = pages => { @@ -31,7 +38,7 @@ export const buildAllPage = pages => { const parts = getSortedHeadNodes(modules).map(({ api }) => byApi.get(api)); return { - data: createSyntheticHead('all', 'All'), + data: createAllPageHead(), headings: parts.flatMap(({ headings }) => headings), parts: parts.map(({ data }) => data.api), }; diff --git a/packages/react/src/html/utils/generate.mjs b/packages/react/src/html/utils/generate.mjs index c8b0def57..6f3a8bce4 100644 --- a/packages/react/src/html/utils/generate.mjs +++ b/packages/react/src/html/utils/generate.mjs @@ -118,7 +118,7 @@ export default () => { * `content` of the pages it is made of, so their JSX is never rebuilt. * * @param {import('../types').Page} page - * @param {string} libraryURL - Where the built component library was written. + * @param {string} libraryURL - The `file:` URL the page imports the component library from. * @returns {string} The program, as JSX. */ const buildPageProgram = (page, libraryURL) => { diff --git a/packages/react/src/html/utils/render.mjs b/packages/react/src/html/utils/render.mjs index 4758e9c62..25787ed70 100644 --- a/packages/react/src/html/utils/render.mjs +++ b/packages/react/src/html/utils/render.mjs @@ -74,24 +74,39 @@ export const processChunk = async (tasks, indices, { template, assets }) => { }; /** - * Creates the page writer: it spreads the pages across the worker pool and - * waits for every one to be written. Without a pool — the generator was called - * directly rather than by the orchestrator — the pages are rendered on the - * calling thread instead. + * Renders each batch of pages on the calling thread, as it arrives. + * + * @param {AsyncIterable>} batches + * @param {{ template: string, assets: import('../types').ClientAssets }} extra + * @yields {Array} Each batch's written file names + */ +async function* renderOnThisThread(batches, extra) { + for await (const tasks of batches) { + const written = await processChunk(tasks, [...tasks.keys()], extra); + + yield written; + } +} + +/** + * Creates the page writer: it spreads the pages across the worker pool as they + * arrive, in batches, and waits for every one to be written. Without a pool — + * the generator was called directly rather than by the orchestrator — the + * pages are rendered on the calling thread instead. * * @param {ParallelWorker} [worker] - * @returns {(tasks: Array, extra: { template: string, assets: import('../types').ClientAssets }) => Promise} + * @returns {(batches: AsyncIterable>, extra: { template: string, assets: import('../types').ClientAssets }) => Promise} */ -export const createPageWriter = worker => async (tasks, extra) => { +export const createPageWriter = worker => async (batches, extra) => { const chunks = worker - ? worker.stream(tasks, extra) - : [await processChunk(tasks, [...tasks.keys()], extra)]; + ? worker.stream(batches, extra) + : renderOnThisThread(batches, extra); let count = 0; for await (const chunk of chunks) { count += chunk.length; - renderLogger.debug(`Wrote ${count}/${tasks.length} pages`); + renderLogger.debug(`Wrote ${count} pages`); } }; diff --git a/packages/react/src/jsx-ast/README.md b/packages/react/src/jsx-ast/README.md index aab9b9f8a..1efdeaa76 100644 --- a/packages/react/src/jsx-ast/README.md +++ b/packages/react/src/jsx-ast/README.md @@ -22,6 +22,18 @@ the content — the `html` generator wraps each page in ``, and assemble `all.html` from the module pages' content (see its `generateAllPage` option) rather than building every module a second time here. +Pages are emitted as they are built, in the order their entries arrived, the +synthetic pages last. Once every page has arrived, it also emits +`{ manifest }`, every page's head in that same order, so that the `html` +generator can build what needs the whole site while pages are still being +built. + +To list the pages as the site does, leave out the `{ manifest }` item and sort +the pages by their head with `headingSortFn`, from +`@doc-kit/generator-react/jsx-ast/utils/getSortedHeadNodes`, as +`pages.toSorted((a, b) => headingSortFn(a.data, b.data))`, keeping the +synthetic pages (`data.synthetic`) last. + ## Index page `index.html` is generated when an `index` document is part of the input, and diff --git a/packages/react/src/jsx-ast/__tests__/generate.test.mjs b/packages/react/src/jsx-ast/__tests__/generate.test.mjs index d0324af46..51a0ecb96 100644 --- a/packages/react/src/jsx-ast/__tests__/generate.test.mjs +++ b/packages/react/src/jsx-ast/__tests__/generate.test.mjs @@ -1,5 +1,7 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; +import { setFlagsFromString } from 'node:v8'; +import { runInNewContext } from 'node:vm'; import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; import getConfig, { @@ -9,6 +11,11 @@ import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; import { generate, processChunk } from '../generate.mjs'; +setFlagsFromString('--expose-gc'); + +// Collects garbage, to tell what is let go +const gc = runInNewContext('gc'); + // Pages are rendered with the pipeline of `jsx-ast` await loadMarkdownPlugins( await loadGenerator(import.meta.resolve('../index.mjs')) @@ -49,23 +56,54 @@ const createEntry = (api, name, { stabilityIndex = '2' } = {}) => { }; }; -const collect = async generator => { - const results = []; +/** + * Reads everything the generator emits, keeping the pages apart from the + * manifests. + * + * @param {AsyncIterable>} output + * @returns {Promise<{ pages: Array, manifests: Array> }>} + */ +const collect = async output => { + const pages = []; + const manifests = []; - for await (const chunk of generator) { - results.push(...chunk); + for await (const items of output) { + for (const item of items) { + if ('manifest' in item) { + manifests.push(item.manifest); + } else { + pages.push(item); + } + } } - return results; + return { pages, manifests }; }; +/** + * Stands in for the worker pool: builds each page from its head alone, as its + * batch arrives, and records every item it is given. + * + * @param {Array} seenItems - Where the items it is given go + */ const createWorker = seenItems => ({ - async *stream(items) { - seenItems.push(...items); - yield items.map(({ head }) => ({ type: 'JSXElement', data: head })); + async *stream(batches) { + for await (const items of batches) { + seenItems.push(...items); + yield items.map(({ head }) => ({ type: 'JSXElement', data: head })); + } }, }); +/** + * Yields each batch in turn, as a streamed input. + * + * @param {Array>} batches + */ +async function* streamOf(batches) { + yield* batches; +} + describe('jsx-ast generate', () => { it('returns the page content as a JSX fragment alongside its ToC', async () => { await setConfig({ target: ['jsx-ast'] }); @@ -92,9 +130,11 @@ describe('jsx-ast generate', () => { getConfig('jsx-ast').generateNotFoundPage = false; const seenItems = []; - const results = await collect( + const { pages, manifests } = await collect( generate( - [createEntry('index', 'Index'), createEntry('fs', 'File system')], + streamOf([ + [createEntry('index', 'Index'), createEntry('fs', 'File system')], + ]), createWorker(seenItems) ) ); @@ -104,11 +144,181 @@ describe('jsx-ast generate', () => { ['index', 'fs'] ); assert.deepEqual( - results.map(({ data }) => data.api), + pages.map(({ data }) => data.api), ['index', 'fs'] ); + + // Without the 404 page, the manifest lists only the input's pages + assert.deepEqual( + manifests.map(heads => heads.map(({ api }) => api)), + [['index', 'fs']] + ); + }); + + it('starts pages as they arrive and emits them as they are built', async () => { + await setConfig({ target: ['jsx-ast'] }); + + const seenItems = []; + const { pages, manifests } = await collect( + generate( + streamOf([ + [createEntry('fs', 'File system')], + [createEntry('buffer', 'Buffer'), createEntry('index', 'Index')], + ]), + createWorker(seenItems) + ) + ); + + // Built in the order they arrived, the 404 page last + assert.deepEqual( + seenItems.map(({ head }) => head.api), + ['fs', 'buffer', 'index', '404'] + ); + + // Emitted as built: the html generator puts them in the site's order + assert.deepEqual( + pages.map(({ data }) => data.api), + ['fs', 'buffer', 'index', '404'] + ); + + // Once every page has arrived, every page's head, in that order + assert.deepEqual( + manifests.map(heads => heads.map(({ api }) => api)), + [['fs', 'buffer', 'index', '404']] + ); + }); + + it('builds the same pages from its whole input, as it gets it with one thread', async () => { + await setConfig({ target: ['jsx-ast'] }); + + const entries = () => [ + createEntry('fs', 'File system'), + createEntry('buffer', 'Buffer'), + ]; + + const streamed = await collect( + generate(streamOf([entries()]), createWorker([])) + ); + const whole = await collect(generate(entries(), createWorker([]))); + + assert.deepEqual(whole, streamed); + + // The manifest still comes with them + assert.equal(whole.manifests.length, 1); }); + it( + 'emits the manifest without waiting for the pages still being built', + // The first page is built only once the manifest has arrived, so a + // manifest that waited for it would never come + { timeout: 10_000 }, + async () => { + await setConfig({ target: ['jsx-ast'] }); + + const gate = Promise.withResolvers(); + const worker = { + async *stream(batches) { + const all = []; + + // Like the pool, every batch is taken as it arrives + for await (const items of batches) { + all.push(items); + } + + for (const items of all) { + // The first page takes until the test lets it finish + if (items.some(({ head }) => head.api === 'fs')) { + await gate.promise; + } + + yield items.map(({ head }) => ({ data: head })); + } + }, + }; + + const output = generate( + streamOf([[createEntry('fs', 'File system')]]), + worker + ); + + const { value } = await output.next(); + + assert.deepEqual( + value[0].manifest.map(({ api }) => api), + ['fs', '404'] + ); + + gate.resolve(); + + const { pages } = await collect(output); + + assert.deepEqual( + pages.map(({ data }) => data.api), + ['fs', '404'] + ); + } + ); + + it( + 'lets go of the pages it emits before the manifest', + // The input only ends once the test lets it, so pages that waited for the + // manifest would never come + { timeout: 10_000 }, + async () => { + await setConfig({ target: ['jsx-ast'] }); + + const released = new Set(); + const registry = new FinalizationRegistry(api => released.add(api)); + const end = Promise.withResolvers(); + + // `metadata` is still running, so the manifest waits for it + async function* input() { + yield [createEntry('fs', 'File system')]; + yield [createEntry('zlib', 'Zlib')]; + yield [createEntry('buffer', 'Buffer')]; + await end.promise; + } + + const worker = { + async *stream(batches) { + for await (const items of batches) { + yield items.map(({ head }) => { + const page = { data: head }; + + registry.register(page, head.api); + + return page; + }); + } + }, + }; + + const output = generate(input(), worker); + + // Each module's page is built once the next module arrives, and read as + // `html` reads it, letting go of it once it has it. The last one read may + // still be in a register: the first one must not be held. + await output.next(); + await output.next(); + + const reading = output.next(); + + for (let i = 0; i < 10 && !released.has('fs'); i++) { + await new Promise(resolve => setImmediate(resolve)); + gc(); + } + + const releasedEarly = released.has('fs'); + + end.resolve(); + + await reading; + await collect(output); + + assert.equal(releasedEarly, true); + } + ); + it('does not change its input on the main thread', async () => { await setConfig({ target: ['jsx-ast'] }); diff --git a/packages/react/src/jsx-ast/generate.mjs b/packages/react/src/jsx-ast/generate.mjs index 62e518526..a7418260c 100644 --- a/packages/react/src/jsx-ast/generate.mjs +++ b/packages/react/src/jsx-ast/generate.mjs @@ -1,10 +1,13 @@ import { isMainThread } from 'node:worker_threads'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; -import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; +import { + groupNodesByModule, + groupWholeModules, + merge, +} from '@doc-kit/core/utils/generators.mjs'; import { jsx, toJs } from 'estree-util-to-js'; -import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs'; import { buildNotFoundPage } from './utils/synthetic/404.mjs'; /** @@ -43,25 +46,67 @@ export async function processChunk(slicedInput, itemIndices) { /** * Generates per-page JSX code from API metadata. * + * Each page starts building once all of its module's entries have arrived, and + * is emitted once built, in the order the pages arrived, the synthetic pages + * last: the `html` generator puts them in the order the site lists them. + * + * Once every page has arrived, without waiting for the pages still being + * built, it also emits `{ manifest }`: every page's head, in that same order, + * so that `html` can prepare what needs the whole site in the meantime. + * * @type {import('./types').Generator['generate']} */ export async function* generate(input, worker) { - // Create sliced input: each item contains head + its module's entries - // This avoids sending all 4700+ entries to every worker - const groupedModules = groupNodesByModule(input); - const descriptors = getSortedHeadNodes(input).map(head => ({ - head, - entries: groupedModules.get(head.api), - })); - // `all.html` is not built here: it is the module pages concatenated, so the // `html` generator assembles it from their content instead of building // every module a second time. - if (getConfig('jsx-ast').generateNotFoundPage) { - descriptors.push(buildNotFoundPage()); + const synthetic = getConfig('jsx-ast').generateNotFoundPage + ? [buildNotFoundPage()] + : []; + + const manifest = Promise.withResolvers(); + + /** + * Describes each arriving page by its head and its module's entries, so a + * worker gets only what its page needs, not all 4700+ entries. Once every + * page has arrived, it resolves the manifest and describes the synthetic + * pages. + * + * @returns {AsyncGenerator }>>} + */ + async function* describe() { + const heads = []; + + // A page needs all of its module's entries + for await (const entries of groupWholeModules(input)) { + const groupedModules = groupNodesByModule(entries); + const descriptors = entries + .filter(node => node.heading.depth === 1) + .map(head => ({ head, entries: groupedModules.get(head.api) })); + + heads.push(...descriptors.map(({ head }) => head)); + + yield descriptors; + } + + manifest.resolve([...heads, ...synthetic.map(({ head }) => head)]); + + if (synthetic.length > 0) { + yield synthetic; + } } - for await (const chunkResult of worker.stream(descriptors)) { - yield chunkResult; + /** + * Gives the manifest, once every page has arrived. + * + * @returns {AsyncGenerator }>>} + */ + async function* announce() { + const heads = await manifest.promise; + + yield [{ manifest: heads }]; } + + // The manifest goes out as soon as it's ready, not after the next pages + yield* merge(worker.stream(describe()), announce()); } diff --git a/packages/react/src/jsx-ast/index.mjs b/packages/react/src/jsx-ast/index.mjs index 59a9ff298..01c17339e 100644 --- a/packages/react/src/jsx-ast/index.mjs +++ b/packages/react/src/jsx-ast/index.mjs @@ -26,6 +26,8 @@ export default { hasParallelProcessor: true, + streamsInput: true, + markdown: { // The configured remark plugins run before the alerts, so they can make // alerts too diff --git a/packages/react/src/jsx-ast/types.d.ts b/packages/react/src/jsx-ast/types.d.ts index de5771abf..9e98b3957 100644 --- a/packages/react/src/jsx-ast/types.d.ts +++ b/packages/react/src/jsx-ast/types.d.ts @@ -11,6 +11,9 @@ export type Generator = GeneratorMetadata< generateNotFoundPage: boolean; showReadingTime: boolean; }, - Generate, AsyncGenerator>, + Generate< + Array | AsyncIterable>, + AsyncGenerator }>> + >, ProcessChunk<{ head: MetadataEntry; entries: Array }, PageCode> >; diff --git a/packages/react/src/jsx-ast/utils/getSortedHeadNodes.mjs b/packages/react/src/jsx-ast/utils/getSortedHeadNodes.mjs index 3c5d7f049..5623c5150 100644 --- a/packages/react/src/jsx-ast/utils/getSortedHeadNodes.mjs +++ b/packages/react/src/jsx-ast/utils/getSortedHeadNodes.mjs @@ -3,12 +3,13 @@ import { OVERRIDDEN_POSITIONS } from '../constants.mjs'; /** - * Sorts entries by OVERRIDDEN_POSITIONS and then heading name. + * Compares two entries to sort them: those in OVERRIDDEN_POSITIONS first, in + * that order, then the rest by heading name. * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} a * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} b * @returns {number} */ -const headingSortFn = (a, b) => { +export const headingSortFn = (a, b) => { const ai = OVERRIDDEN_POSITIONS.indexOf(a.api); const bi = OVERRIDDEN_POSITIONS.indexOf(b.api);