From 280a8b5c1a57ae2ca715098ef5c8f934007a7638 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Tue, 3 Nov 2026 19:06:50 +0100 Subject: [PATCH 1/2] perf: highlight with fewer grammars by default Each grammar costs every thread its own memory, about 60MB for JavaScript's on Oniguruma, and time to register. Types and signatures are already highlighted as TypeScript, whose grammar highlights JavaScript too, so `js`, `mjs`, `cjs` and `javascript` code is now highlighted with it by default, and C++ (`cpp`, `c++`) with C's. On Node.js's docs that saves ~190MB of peak memory and 2-7% of build time. In the light theme, the default, JavaScript comes out the same but for one block of console output marked as `js`. Nord, the dark theme, has a few colors only for JavaScript's scopes. C++-only keywords lose their color. `langAlias` takes precedence, and doc-kit now resolves aliases itself: Shiki replaced configured aliases with a grammar's own as it registered. Aliases now also lead on through other aliases and are found in any case. Transformers, and the language `resolveLanguage` returns, are the one code is highlighted as. Assisted-by: Claude Opus 5.5 Assisted-by: Claude Sonnet 5.5 --- .changeset/highlight-fewer-grammars.md | 5 ++ docs/configuration.md | 12 ++- docs/creating-generators.md | 4 +- .../shiki/__tests__/highlighter.test.mjs | 87 +++++++++++++++++-- packages/core/src/plugins/shiki/constants.mjs | 17 ++++ .../core/src/plugins/shiki/highlighter.mjs | 45 +++++++--- 6 files changed, 150 insertions(+), 20 deletions(-) create mode 100644 .changeset/highlight-fewer-grammars.md create mode 100644 packages/core/src/plugins/shiki/constants.mjs diff --git a/.changeset/highlight-fewer-grammars.md b/.changeset/highlight-fewer-grammars.md new file mode 100644 index 000000000..0519b6ee3 --- /dev/null +++ b/.changeset/highlight-fewer-grammars.md @@ -0,0 +1,5 @@ +--- +'@doc-kit/core': minor +--- + +Code is highlighted with fewer grammars by default, as each one takes its own memory on every thread: JavaScript (`js`, `mjs`, `cjs`, `javascript`) with TypeScript's grammar, which already highlights types, and C++ (`cpp`, `c++`) with C's. The Shiki plugin's `langAlias` option takes precedence over these defaults, so `{ cpp: 'cpp', 'c++': 'cpp' }` brings back C++'s own grammar, and aliases now keep working once a grammar with the same aliases is registered. A highlighter's `resolveLanguage` now returns the language code is highlighted as, as its Shiki instance names it (`typescript` for `mjs`): highlight with what it returns. Transformers see that language too. diff --git a/docs/configuration.md b/docs/configuration.md index a0d750dd9..53d5c2cfd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -219,12 +219,20 @@ bundles. To configure it, list it with options in dependencies, so your configuration file can resolve it): - `langs` {Array} More languages: grammars, or modules default-exporting them. -- `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`. +- `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`. An + alias leads to the language it names, or on to the one that language is an + alias of. By default, JavaScript (`javascript`, `js`, `mjs`, `cjs`) is + highlighted with TypeScript's grammar, and C++ (`cpp`, `c++`) with C's: + every grammar takes its own memory on each thread, and those highlight that + code too. Aliases take precedence over these defaults: + `{ cpp: 'cpp', 'c++': 'cpp' }` highlights C++ with its own grammar, and a + grammar in `langs` for one of these languages needs an alias to it. - `themes` {Object} The `light` and `dark` themes: names of themes Shiki bundles, modules default-exporting themes, or themes. - `transformers` {Array} Modules default-exporting [Shiki transformers](https://shiki.style/guide/transformers), or lists of - them. + them. They see the language code is highlighted as (`options.lang`), such as + `typescript` for `mjs`. These modules are paths relative to the working directory, or URLs, such as the ones `import.meta.resolve()` gives: diff --git a/docs/creating-generators.md b/docs/creating-generators.md index c2912d335..c30ff1b73 100644 --- a/docs/creating-generators.md +++ b/docs/creating-generators.md @@ -526,7 +526,9 @@ returning the plugin, instead of a default export. The Shiki plugin does, and returns the highlighter of a generator's Shiki plugin, to highlight code as its pipeline does. It registers Shiki's bundled languages as code in them is highlighted, so before using its `shiki` instance directly, pass the language -through its `resolveLanguage`. +through its `resolveLanguage`, and highlight with the language it returns, +which may not be the one the code names (`mjs` gives `typescript`, and an +unknown language `text`). ## File Output diff --git a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs index 030f5d108..7820ad8d8 100644 --- a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs +++ b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs @@ -93,20 +93,97 @@ describe('createHighlighter', () => { const loaded = () => highlighter.shiki.getLoadedLanguages(); assert.ok(!loaded().includes('python')); - assert.ok(!loaded().includes('javascript')); + assert.ok(!loaded().includes('rust')); // By its name, an alias of its own, or one of the options - assert.equal(highlighter.resolveLanguage('py'), 'py'); - assert.equal(highlighter.resolveLanguage('mjs'), 'mjs'); + assert.equal(highlighter.resolveLanguage('py'), 'python'); + assert.equal(highlighter.resolveLanguage('rs'), 'rs'); assert.ok(loaded().includes('python')); - assert.ok(loaded().includes('javascript')); - assert.ok(loaded().includes('cjs')); + assert.ok(loaded().includes('rust')); assert.equal(highlighter.resolveLanguage(undefined), 'text'); assert.equal(highlighter.resolveLanguage('plaintext'), 'plaintext'); }); + it('highlights JavaScript as TypeScript, and C++ as C', async () => { + const highlighter = await createHighlighter(); + const loaded = () => highlighter.shiki.getLoadedLanguages(); + + for (const language of ['javascript', 'JS', 'mjs', 'cjs']) { + assert.equal(highlighter.resolveLanguage(language), 'typescript'); + } + + assert.equal(highlighter.resolveLanguage('cpp'), 'c'); + assert.equal(highlighter.resolveLanguage('c++'), 'c'); + assert.ok(!loaded().includes('cpp')); + + // Even once a grammar that has them as its own aliases is registered + assert.equal(highlighter.resolveLanguage('coffee'), 'coffee'); + assert.ok(loaded().includes('javascript')); + assert.equal(highlighter.resolveLanguage('js'), 'typescript'); + + // Not the names every object inherits + assert.equal(highlighter.resolveLanguage('constructor'), 'text'); + }); + + it('takes aliases over the default ones', async () => { + const highlighter = await createHighlighter({ + langAlias: { cpp: 'cpp', js: 'javascript' }, + }); + + assert.equal(highlighter.resolveLanguage('cpp'), 'cpp'); + assert.equal(highlighter.resolveLanguage('js'), 'javascript'); + assert.equal(highlighter.resolveLanguage('mjs'), 'typescript'); + assert.ok(highlighter.shiki.getLoadedLanguages().includes('cpp')); + }); + + it('follows configured aliases, and finds languages named in any case', async () => { + const highlighter = await createHighlighter({ + langs: [{ ...grammar, name: 'OxcConf', scopeName: 'source.oxcconf2' }], + langAlias: { + a: 'b', + b: 'python', + loop: 'pool', + pool: 'loop', + oc: 'OxcConf', + conf: 'INI', + }, + }); + + assert.equal(highlighter.resolveLanguage('a'), 'python'); + assert.equal(highlighter.resolveLanguage('loop'), 'text'); + assert.equal(highlighter.resolveLanguage('Python'), 'python'); + assert.equal(highlighter.resolveLanguage('oc'), 'OxcConf'); + assert.equal(highlighter.resolveLanguage('conf'), 'ini'); + assert.equal( + highlighter.highlightToHtml('[section]', 'conf'), + highlighter.highlightToHtml('[section]', 'ini') + ); + }); + + it('highlights with a language of `langs` an alias leads to', async () => { + const highlighter = await createHighlighter({ + langs: [ + { + ...grammar, + name: 'javascript', + aliases: ['js'], + scopeName: 'source.oxc-js', + }, + ], + langAlias: { js: 'javascript' }, + }); + + assert.equal(highlighter.resolveLanguage('js'), 'javascript'); + assert.match( + highlighter.highlightToHtml('minify on', 'js'), + /--shiki-dark/ + ); + assert.ok(!highlighter.shiki.getLoadedLanguages().includes('typescript')); + assert.equal(highlighter.resolveLanguage('mjs'), 'typescript'); + }); + it('lists the bundled languages without registering them', async () => { const highlighter = await createHighlighter({ langs: [grammar] }); diff --git a/packages/core/src/plugins/shiki/constants.mjs b/packages/core/src/plugins/shiki/constants.mjs new file mode 100644 index 000000000..2d5e9cc75 --- /dev/null +++ b/packages/core/src/plugins/shiki/constants.mjs @@ -0,0 +1,17 @@ +'use strict'; + +// What code in an unknown language is highlighted as +export const FALLBACK_LANGUAGE = 'text'; + +// What code in these languages is highlighted as, unless configured otherwise. +// Every grammar takes its own memory on each thread, and time to register, and +// TypeScript's highlights JavaScript (types are highlighted as TypeScript +// anyway), as C's does most of C++ +export const DEFAULT_ALIASES = new Map([ + ['javascript', 'typescript'], + ['js', 'typescript'], + ['mjs', 'typescript'], + ['cjs', 'typescript'], + ['cpp', 'c'], + ['c++', 'c'], +]); diff --git a/packages/core/src/plugins/shiki/highlighter.mjs b/packages/core/src/plugins/shiki/highlighter.mjs index 06dc0a582..bf9f74432 100644 --- a/packages/core/src/plugins/shiki/highlighter.mjs +++ b/packages/core/src/plugins/shiki/highlighter.mjs @@ -12,6 +12,8 @@ import { importFromURL } from '#utils/loaders.mjs'; import { getMarkdownPlugins } from '#utils/markdown/plugins.mjs'; import { lazy } from '#utils/misc.mjs'; +import { DEFAULT_ALIASES, FALLBACK_LANGUAGE } from './constants.mjs'; + const require = createRequire(import.meta.url); /** @@ -25,7 +27,7 @@ const require = createRequire(import.meta.url); * * @typedef {Object} SyntaxHighlighter * @property {import('shiki').HighlighterCore} shiki - The Shiki instance - * @property {(languageId?: string) => string} resolveLanguage - Resolves a language, falling back to plain text for unknown ones + * @property {(languageId?: string) => string} resolveLanguage - Resolves the language code is highlighted as, falling back to plain text for unknown ones * @property {(code: string, lang: string, meta?: Record) => string} highlightToHtml - Highlights code, returning the inner HTML of its `` element * @property {(code: string, lang: string, meta?: Record) => ReturnType} highlightToHast - Highlights code, returning a HAST tree * @property {Array} langs - The languages it highlights @@ -64,9 +66,6 @@ let engine; // The highlighters of the options given, by their JSON const highlighters = new Map(); -// What code in an unknown language is highlighted as -const FALLBACK_LANGUAGE = 'text'; - // The languages Shiki bundles. Registering a grammar takes time and memory, // and slows down every highlight after it, so each one is only imported and // registered once code in it is highlighted (see `resolveLanguage`). @@ -220,10 +219,12 @@ const importHighlighter = async ({ engine: regexEngine, // The bundled languages are registered as code uses them langs: withEmbeddedLanguages(importedLangs), - // A copy, as Shiki adds the aliases of the languages it registers to it - langAlias: { ...langAlias }, }; + // Resolved here, as Shiki replaces an alias with a grammar's own as it + // registers + const aliases = new Map(Object.entries(langAlias)); + // Without themes of its own, the highlighter has a default light and dark one if (themes) { const [light, dark] = await Promise.all([ @@ -256,8 +257,9 @@ const importHighlighter = async ({ }); /** - * Resolves a language, falling back to plain text for unknown ones. A - * bundled language is registered the first time it's resolved. + * Resolves the language code is highlighted as, through the aliases, falling + * back to plain text for unknown ones. A bundled language is registered the + * first time it's resolved. * * @param {string} [languageId] * @returns {string} @@ -267,20 +269,39 @@ const importHighlighter = async ({ return FALLBACK_LANGUAGE; } + const followed = new Set(); + + let language = languageId; + + // Each configured alias once, so circular ones end + while (aliases.has(language.toLowerCase()) && !followed.has(language)) { + followed.add(language); + language = aliases.get(language.toLowerCase()); + } + + if (followed.size === 0) { + language = DEFAULT_ALIASES.get(language.toLowerCase()) ?? language; + } + const instance = load().shiki; - const name = instance.resolveLangAlias(languageId.toLowerCase()); + const loaded = instance.getLoadedLanguages(); - if (isSpecialLang(name) || instance.getLoadedLanguages().includes(name)) { - return languageId; + // Shiki has bundled languages' names and aliases in lowercase + for (const name of [language, language.toLowerCase()]) { + if (isSpecialLang(name) || loaded.includes(name)) { + return name; + } } + const name = language.toLowerCase(); + if (!BUNDLED_NAMES.has(name)) { return FALLBACK_LANGUAGE; } instance.loadLanguageSync(importBundledLanguage(name)); - return languageId; + return name; }; /** From ad1a970cbd58514be3b08dbafe3b21d89ef8b5fc Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Wed, 4 Nov 2026 19:08:41 +0100 Subject: [PATCH 2/2] fix: match configured aliases with capitals in any case too The lookup lowercased the language, but the aliases kept their keys as written, so `{ MyConf: 'ini' }` never matched. They're keyed in lowercase now. The test also says why Rust stands in for a bundled language registered through its own alias. Assisted-by: Claude Opus 5.5 --- .../core/src/plugins/shiki/__tests__/highlighter.test.mjs | 5 +++++ packages/core/src/plugins/shiki/highlighter.mjs | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs index 7820ad8d8..2132aecec 100644 --- a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs +++ b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs @@ -92,6 +92,8 @@ describe('createHighlighter', () => { }); const loaded = () => highlighter.shiki.getLoadedLanguages(); + // Rust stands for any bundled language nothing aliases: JavaScript's + // aliases now lead to TypeScript assert.ok(!loaded().includes('python')); assert.ok(!loaded().includes('rust')); @@ -148,6 +150,7 @@ describe('createHighlighter', () => { pool: 'loop', oc: 'OxcConf', conf: 'INI', + MyConf: 'ini', }, }); @@ -156,6 +159,8 @@ describe('createHighlighter', () => { assert.equal(highlighter.resolveLanguage('Python'), 'python'); assert.equal(highlighter.resolveLanguage('oc'), 'OxcConf'); assert.equal(highlighter.resolveLanguage('conf'), 'ini'); + assert.equal(highlighter.resolveLanguage('myconf'), 'ini'); + assert.equal(highlighter.resolveLanguage('MYCONF'), 'ini'); assert.equal( highlighter.highlightToHtml('[section]', 'conf'), highlighter.highlightToHtml('[section]', 'ini') diff --git a/packages/core/src/plugins/shiki/highlighter.mjs b/packages/core/src/plugins/shiki/highlighter.mjs index bf9f74432..b0fdc5602 100644 --- a/packages/core/src/plugins/shiki/highlighter.mjs +++ b/packages/core/src/plugins/shiki/highlighter.mjs @@ -222,8 +222,10 @@ const importHighlighter = async ({ }; // Resolved here, as Shiki replaces an alias with a grammar's own as it - // registers - const aliases = new Map(Object.entries(langAlias)); + // registers. Keyed in lowercase, as they're matched in any case + const aliases = new Map( + Object.entries(langAlias).map(([from, to]) => [from.toLowerCase(), to]) + ); // Without themes of its own, the highlighter has a default light and dark one if (themes) {