perf: let jsx-ast and html start on their dependency's first chunk

.changeset/stream-pages.md

@@ -0,0 +1,6 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+---
+
+`jsx-ast` and `html` stream their input: with more than one thread, page building and rendering overlap with the generators before them. `html` bundles as soon as every page's metadata is known, and renders pages as they are built. Streaming leaves the generated files unchanged, but `jsx-ast` now yields pages in the order their entries arrive, not the order the site lists them, and, once every page has arrived, a `{ manifest }` item listing their heads in that order. To list the pages as the site does, skip the `{ manifest }` item and sort them with `headingSortFn`, now exported from `@doc-kit/generator-react/jsx-ast/utils/getSortedHeadNodes`, as `pages.toSorted((a, b) => headingSortFn(a.data, b.data))`, keeping the synthetic pages (`data.synthetic`) last. Bundler adapters' `compile` can now be called before and while `buildServer` and `buildClient` run. `createChannel` and `merge`, from `@doc-kit/core/utils`, are the queue and the merge of async iterables they are built on.

packages/core/src/utils/__tests__/channel.test.mjs

@@ -0,0 +1,63 @@
+import { deepStrictEqual } from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { createChannel } from '../channel.mjs';
+
+/**
+ * Reads every batch of an async iterable.
+ *
+ * @template T
+ * @param {AsyncIterable<T[]>} iterable
+ * @returns {Promise<T[][]>}
+ */
+async function readBatches(iterable) {
+  const batches = [];
+
+  for await (const batch of iterable) {
+    batches.push(batch);
+  }
+
+  return batches;
+}
+
+describe('createChannel', () => {
+  it('gives the batches as they are added, until it ends', async () => {
+    const channel = createChannel();
+    const reading = readBatches(channel);
+
+    channel.push([1]);
+    await new Promise(resolve => setImmediate(resolve));
+    channel.push([2]);
+    channel.end();
+
+    const batches = await reading;
+
+    deepStrictEqual(batches, [[1], [2]]);
+  });
+
+  it('gives a prepended batch before those not read yet', async () => {
+    const channel = createChannel();
+
+    channel.push([1]);
+    channel.push([2]);
+    channel.prepend([0]);
+    channel.end();
+
+    const batches = await readBatches(channel);
+
+    deepStrictEqual(batches, [[0], [1], [2]]);
+  });
+
+  it('drops the batches not read yet when cleared', async () => {
+    const channel = createChannel();
+
+    channel.push([1]);
+    channel.clear();
+    channel.push([2]);
+    channel.end();
+
+    const batches = await readBatches(channel);
+
+    deepStrictEqual(batches, [[2]]);
+  });
+});

packages/core/src/utils/__tests__/generators.test.mjs

@@ -8,6 +8,7 @@ import {
   getVersionFromSemVer,
   coerceSemVer,
   getCompatibleVersions,
+  merge,
   toBatches,
 } from '../generators.mjs';
 
@@ -244,3 +245,56 @@ describe('toBatches', () => {
     assert.equal(toBatches(streamed), streamed);
   });
 });
+
+describe('merge', () => {
+  it('yields each item as soon as it arrives, until every source ends', async () => {
+    const gate = Promise.withResolvers();
+
+    async function* slow() {
+      await gate.promise;
+
+      yield 'slow';
+    }
+
+    async function* fast() {
+      yield 'fast';
+      yield 'faster';
+    }
+
+    const items = [];
+
+    for await (const item of merge(slow(), fast())) {
+      items.push(item);
+
+      if (item === 'faster') {
+        gate.resolve();
+      }
+    }
+
+    assert.deepEqual(items, ['fast', 'faster', 'slow']);
+  });
+
+  it('throws a failure without waiting for the other sources', async () => {
+    async function* never() {
+      await new Promise(() => {});
+
+      yield 'never';
+    }
+
+    async function* failing() {
+      yield 'before';
+
+      throw new Error('merged failure');
+    }
+
+    const items = [];
+
+    await assert.rejects(async () => {
+      for await (const item of merge(never(), failing())) {
+        items.push(item);
+      }
+    }, /merged failure/);
+
+    assert.deepEqual(items, ['before']);
+  });
+});

packages/core/src/utils/channel.mjs

@@ -0,0 +1,75 @@
+'use strict';
+
+import { createSignal } from './misc.mjs';
+
+/**
+ * Creates a queue of batches that can be read as an async iterable while it is
+ * still being filled.
+ *
+ * @template T
+ */
+export const createChannel = () => {
+  /** @type {Array<T[]>} */
+  const batches = [];
+
+  // Notified as batches are added, and at the end
+  const added = createSignal();
+
+  let ended = false;
+
+  return {
+    /**
+     * Adds a batch, after those not read yet.
+     *
+     * @param {T[]} batch
+     */
+    push(batch) {
+      batches.push(batch);
+      added.notify();
+    },
+
+    /**
+     * Adds a batch, before those not read yet.
+     *
+     * @param {T[]} batch
+     */
+    prepend(batch) {
+      batches.unshift(batch);
+      added.notify();
+    },
+
+    /**
+     * Drops the batches not read yet.
+     */
+    clear() {
+      batches.length = 0;
+    },
+
+    /**
+     * Ends the channel: the reader stops once it has read every batch.
+     */
+    end() {
+      ended = true;
+      added.notify();
+    },
+
+    /**
+     * Reads the batches as they are added, until the channel ends.
+     *
+     * @returns {AsyncGenerator<T[]>}
+     */
+    async *[Symbol.asyncIterator]() {
+      for (;;) {
+        while (batches.length === 0) {
+          if (ended) {
+            return;
+          }
+
+          await added.wait();
+        }
+
+        yield batches.shift();
+      }
+    },
+  };
+};

packages/core/src/utils/generators.mjs

@@ -2,6 +2,7 @@
 
 import { coerce, major } from 'semver';
 
+import { createChannel } from './channel.mjs';
 import { isAsyncIterable } from './misc.mjs';
 import { transformNodeToString } from './unist.mjs';
 
@@ -59,6 +60,54 @@ export const toBatches = input => {
   return [input];
 };
 
+/**
+ * Reads several async iterables at once, yielding each item as soon as it
+ * arrives, until they have all ended. A failure is thrown once the items that
+ * arrived before it are yielded, without waiting for the other iterables.
+ *
+ * @template T
+ * @param {...AsyncIterable<T>} sources
+ * @returns {AsyncGenerator<T>}
+ */
+export async function* merge(...sources) {
+  const channel = createChannel();
+
+  let pending = sources.length;
+  let failure;
+
+  /**
+   * Passes a source's items on as they arrive, and ends the channel once every
+   * source has ended, or one fails.
+   *
+   * @param {AsyncIterable<T>} source
+   */
+  const pipe = async source => {
+    try {
+      for await (const item of source) {
+        channel.push(item);
+      }
+    } catch (error) {
+      failure ??= { error };
+    } finally {
+      pending--;
+
+      if (failure || pending === 0) {
+        channel.end();
+      }
+    }
+  };
+
+  for (const source of sources) {
+    pipe(source);
+  }
+
+  yield* channel;
+
+  if (failure) {
+    throw failure.error;
+  }
+}
+
 /**
  * Regroups streamed entries so that each batch holds whole modules: a module
  * whose entries continue in the next batch is held back and passed on with

packages/react/package.json

@@ -15,6 +15,7 @@
     "./html/ui/*": "./src/html/ui/*",
     "./html/bundlers/vite": "./src/html/bundlers/vite.mjs",
     "./jsx-ast": "./src/jsx-ast/index.mjs",
+    "./jsx-ast/utils/getSortedHeadNodes": "./src/jsx-ast/utils/getSortedHeadNodes.mjs",
     "./llms-txt": "./src/llms-txt/index.mjs",
     "./llms-txt-full": "./src/llms-txt-full/index.mjs",
     "./orama-db": "./src/orama-db/index.mjs",

packages/react/src/html/README.md

@@ -6,12 +6,15 @@ assets, written to `output`. Vite is the default bundler adapter, but projects
 can supply an adapter for webpack or another bundler. The generator is
 output-only and does not return an in-memory copy of its HTML or CSS.
 
-The site is built in pieces that are each as small as they can be. The bundler
-builds the component library once and the client assets once. Each page's
-program is then compiled — JSX to a plain module — and the worker pool imports,
-renders, templates, minifies and writes the pages one at a time, so memory
-scales with the largest page rather than with the site. `all.html` is assembled
-from the module pages' compiled content rather than built again from scratch.
+The site is built in pieces that are each as small as they can be, each as
+soon as what it needs is there. Each page's program is compiled — JSX to a
+plain module — as `jsx-ast` builds the page. The bundler builds the component
+library once and the client assets once, as soon as every page's metadata is
+known, and the worker pool imports, renders, templates, minifies and writes
+the pages one at a time as they are compiled, so memory scales with the largest
+page rather than with the site. With one thread, each step waits for the one
+before it to finish. `all.html` is assembled from the module pages' compiled
+content rather than built again from scratch.
 
 ## Configuring
 
@@ -201,16 +204,21 @@ for a particular build system.
 `buildServer` receives `{ entry, virtualImports, outDir, config }`. The `entry`
 is the component library's source: re-exports of every component a page may
 render, Preact's `h` and `Fragment`, and `renderToStringAsync`. It must be
-bundled into one self-contained module written under `outDir` (a temporary
-directory the generator removes afterwards), since the page programs import it
-from wherever they are compiled to.
+bundled into one self-contained module written under `outDir` (an empty
+temporary directory the generator removes afterwards), since the page programs
+import it from wherever they are compiled to.
 
 `compile(code, fileName)` receives one page program: a module that imports the
 library and exports the page's `content` and a default function rendering the
-page from its layout props, written in JSX. It must return plain JavaScript. The JSX must compile with the classic
-runtime to calls of the `_jsx` and `_Fragment` bindings the program imports
-(these names are exported as `JSX_PRAGMA` and `JSX_PRAGMA_FRAG` from the
-generator's `constants.mjs`), so that the page and the library share one Preact.
+page from its layout props, written in JSX. It must return plain JavaScript.
+Pages are compiled as they are built, so `compile` can be called before
+`buildServer`, and while `buildServer` or `buildClient` still runs: it must
+leave the program's imports as they are, since the library they point to may
+not exist yet, and must not depend on the builds. The JSX must compile with the
+classic runtime to calls of the `_jsx` and `_Fragment` bindings the program
+imports (these names are exported as `JSX_PRAGMA` and `JSX_PRAGMA_FRAG` from
+the generator's `constants.mjs`), so that the page and the library share one
+Preact.
 
 `buildClient` receives `{ entry, virtualImports, config }`. The client `entry`
 is a single program shared by every page. It must be bundled into