diff --git a/.changeset/highlight-fewer-grammars.md b/.changeset/highlight-fewer-grammars.md new file mode 100644 index 000000000..0519b6ee3 --- /dev/null +++ b/.changeset/highlight-fewer-grammars.md @@ -0,0 +1,5 @@ +--- +'@doc-kit/core': minor +--- + +Code is highlighted with fewer grammars by default, as each one takes its own memory on every thread: JavaScript (`js`, `mjs`, `cjs`, `javascript`) with TypeScript's grammar, which already highlights types, and C++ (`cpp`, `c++`) with C's. The Shiki plugin's `langAlias` option takes precedence over these defaults, so `{ cpp: 'cpp', 'c++': 'cpp' }` brings back C++'s own grammar, and aliases now keep working once a grammar with the same aliases is registered. A highlighter's `resolveLanguage` now returns the language code is highlighted as, as its Shiki instance names it (`typescript` for `mjs`): highlight with what it returns. Transformers see that language too. diff --git a/docs/configuration.md b/docs/configuration.md index a0d750dd9..53d5c2cfd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -219,12 +219,20 @@ bundles. To configure it, list it with options in dependencies, so your configuration file can resolve it): - `langs` {Array} More languages: grammars, or modules default-exporting them. -- `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`. +- `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`. An + alias leads to the language it names, or on to the one that language is an + alias of. By default, JavaScript (`javascript`, `js`, `mjs`, `cjs`) is + highlighted with TypeScript's grammar, and C++ (`cpp`, `c++`) with C's: + every grammar takes its own memory on each thread, and those highlight that + code too. Aliases take precedence over these defaults: + `{ cpp: 'cpp', 'c++': 'cpp' }` highlights C++ with its own grammar, and a + grammar in `langs` for one of these languages needs an alias to it. - `themes` {Object} The `light` and `dark` themes: names of themes Shiki bundles, modules default-exporting themes, or themes. - `transformers` {Array} Modules default-exporting [Shiki transformers](https://shiki.style/guide/transformers), or lists of - them. + them. They see the language code is highlighted as (`options.lang`), such as + `typescript` for `mjs`. These modules are paths relative to the working directory, or URLs, such as the ones `import.meta.resolve()` gives: diff --git a/docs/creating-generators.md b/docs/creating-generators.md index c2912d335..c30ff1b73 100644 --- a/docs/creating-generators.md +++ b/docs/creating-generators.md @@ -526,7 +526,9 @@ returning the plugin, instead of a default export. The Shiki plugin does, and returns the highlighter of a generator's Shiki plugin, to highlight code as its pipeline does. It registers Shiki's bundled languages as code in them is highlighted, so before using its `shiki` instance directly, pass the language -through its `resolveLanguage`. +through its `resolveLanguage`, and highlight with the language it returns, +which may not be the one the code names (`mjs` gives `typescript`, and an +unknown language `text`). ## File Output diff --git a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs index 030f5d108..7820ad8d8 100644 --- a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs +++ b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs @@ -93,20 +93,97 @@ describe('createHighlighter', () => { const loaded = () => highlighter.shiki.getLoadedLanguages(); assert.ok(!loaded().includes('python')); - assert.ok(!loaded().includes('javascript')); + assert.ok(!loaded().includes('rust')); // By its name, an alias of its own, or one of the options - assert.equal(highlighter.resolveLanguage('py'), 'py'); - assert.equal(highlighter.resolveLanguage('mjs'), 'mjs'); + assert.equal(highlighter.resolveLanguage('py'), 'python'); + assert.equal(highlighter.resolveLanguage('rs'), 'rs'); assert.ok(loaded().includes('python')); - assert.ok(loaded().includes('javascript')); - assert.ok(loaded().includes('cjs')); + assert.ok(loaded().includes('rust')); assert.equal(highlighter.resolveLanguage(undefined), 'text'); assert.equal(highlighter.resolveLanguage('plaintext'), 'plaintext'); }); + it('highlights JavaScript as TypeScript, and C++ as C', async () => { + const highlighter = await createHighlighter(); + const loaded = () => highlighter.shiki.getLoadedLanguages(); + + for (const language of ['javascript', 'JS', 'mjs', 'cjs']) { + assert.equal(highlighter.resolveLanguage(language), 'typescript'); + } + + assert.equal(highlighter.resolveLanguage('cpp'), 'c'); + assert.equal(highlighter.resolveLanguage('c++'), 'c'); + assert.ok(!loaded().includes('cpp')); + + // Even once a grammar that has them as its own aliases is registered + assert.equal(highlighter.resolveLanguage('coffee'), 'coffee'); + assert.ok(loaded().includes('javascript')); + assert.equal(highlighter.resolveLanguage('js'), 'typescript'); + + // Not the names every object inherits + assert.equal(highlighter.resolveLanguage('constructor'), 'text'); + }); + + it('takes aliases over the default ones', async () => { + const highlighter = await createHighlighter({ + langAlias: { cpp: 'cpp', js: 'javascript' }, + }); + + assert.equal(highlighter.resolveLanguage('cpp'), 'cpp'); + assert.equal(highlighter.resolveLanguage('js'), 'javascript'); + assert.equal(highlighter.resolveLanguage('mjs'), 'typescript'); + assert.ok(highlighter.shiki.getLoadedLanguages().includes('cpp')); + }); + + it('follows configured aliases, and finds languages named in any case', async () => { + const highlighter = await createHighlighter({ + langs: [{ ...grammar, name: 'OxcConf', scopeName: 'source.oxcconf2' }], + langAlias: { + a: 'b', + b: 'python', + loop: 'pool', + pool: 'loop', + oc: 'OxcConf', + conf: 'INI', + }, + }); + + assert.equal(highlighter.resolveLanguage('a'), 'python'); + assert.equal(highlighter.resolveLanguage('loop'), 'text'); + assert.equal(highlighter.resolveLanguage('Python'), 'python'); + assert.equal(highlighter.resolveLanguage('oc'), 'OxcConf'); + assert.equal(highlighter.resolveLanguage('conf'), 'ini'); + assert.equal( + highlighter.highlightToHtml('[section]', 'conf'), + highlighter.highlightToHtml('[section]', 'ini') + ); + }); + + it('highlights with a language of `langs` an alias leads to', async () => { + const highlighter = await createHighlighter({ + langs: [ + { + ...grammar, + name: 'javascript', + aliases: ['js'], + scopeName: 'source.oxc-js', + }, + ], + langAlias: { js: 'javascript' }, + }); + + assert.equal(highlighter.resolveLanguage('js'), 'javascript'); + assert.match( + highlighter.highlightToHtml('minify on', 'js'), + /--shiki-dark/ + ); + assert.ok(!highlighter.shiki.getLoadedLanguages().includes('typescript')); + assert.equal(highlighter.resolveLanguage('mjs'), 'typescript'); + }); + it('lists the bundled languages without registering them', async () => { const highlighter = await createHighlighter({ langs: [grammar] }); diff --git a/packages/core/src/plugins/shiki/constants.mjs b/packages/core/src/plugins/shiki/constants.mjs new file mode 100644 index 000000000..2d5e9cc75 --- /dev/null +++ b/packages/core/src/plugins/shiki/constants.mjs @@ -0,0 +1,17 @@ +'use strict'; + +// What code in an unknown language is highlighted as +export const FALLBACK_LANGUAGE = 'text'; + +// What code in these languages is highlighted as, unless configured otherwise. +// Every grammar takes its own memory on each thread, and time to register, and +// TypeScript's highlights JavaScript (types are highlighted as TypeScript +// anyway), as C's does most of C++ +export const DEFAULT_ALIASES = new Map([ + ['javascript', 'typescript'], + ['js', 'typescript'], + ['mjs', 'typescript'], + ['cjs', 'typescript'], + ['cpp', 'c'], + ['c++', 'c'], +]); diff --git a/packages/core/src/plugins/shiki/highlighter.mjs b/packages/core/src/plugins/shiki/highlighter.mjs index 06dc0a582..bf9f74432 100644 --- a/packages/core/src/plugins/shiki/highlighter.mjs +++ b/packages/core/src/plugins/shiki/highlighter.mjs @@ -12,6 +12,8 @@ import { importFromURL } from '#utils/loaders.mjs'; import { getMarkdownPlugins } from '#utils/markdown/plugins.mjs'; import { lazy } from '#utils/misc.mjs'; +import { DEFAULT_ALIASES, FALLBACK_LANGUAGE } from './constants.mjs'; + const require = createRequire(import.meta.url); /** @@ -25,7 +27,7 @@ const require = createRequire(import.meta.url); * * @typedef {Object} SyntaxHighlighter * @property {import('shiki').HighlighterCore} shiki - The Shiki instance - * @property {(languageId?: string) => string} resolveLanguage - Resolves a language, falling back to plain text for unknown ones + * @property {(languageId?: string) => string} resolveLanguage - Resolves the language code is highlighted as, falling back to plain text for unknown ones * @property {(code: string, lang: string, meta?: Record) => string} highlightToHtml - Highlights code, returning the inner HTML of its `` element * @property {(code: string, lang: string, meta?: Record) => ReturnType} highlightToHast - Highlights code, returning a HAST tree * @property {Array} langs - The languages it highlights @@ -64,9 +66,6 @@ let engine; // The highlighters of the options given, by their JSON const highlighters = new Map(); -// What code in an unknown language is highlighted as -const FALLBACK_LANGUAGE = 'text'; - // The languages Shiki bundles. Registering a grammar takes time and memory, // and slows down every highlight after it, so each one is only imported and // registered once code in it is highlighted (see `resolveLanguage`). @@ -220,10 +219,12 @@ const importHighlighter = async ({ engine: regexEngine, // The bundled languages are registered as code uses them langs: withEmbeddedLanguages(importedLangs), - // A copy, as Shiki adds the aliases of the languages it registers to it - langAlias: { ...langAlias }, }; + // Resolved here, as Shiki replaces an alias with a grammar's own as it + // registers + const aliases = new Map(Object.entries(langAlias)); + // Without themes of its own, the highlighter has a default light and dark one if (themes) { const [light, dark] = await Promise.all([ @@ -256,8 +257,9 @@ const importHighlighter = async ({ }); /** - * Resolves a language, falling back to plain text for unknown ones. A - * bundled language is registered the first time it's resolved. + * Resolves the language code is highlighted as, through the aliases, falling + * back to plain text for unknown ones. A bundled language is registered the + * first time it's resolved. * * @param {string} [languageId] * @returns {string} @@ -267,20 +269,39 @@ const importHighlighter = async ({ return FALLBACK_LANGUAGE; } + const followed = new Set(); + + let language = languageId; + + // Each configured alias once, so circular ones end + while (aliases.has(language.toLowerCase()) && !followed.has(language)) { + followed.add(language); + language = aliases.get(language.toLowerCase()); + } + + if (followed.size === 0) { + language = DEFAULT_ALIASES.get(language.toLowerCase()) ?? language; + } + const instance = load().shiki; - const name = instance.resolveLangAlias(languageId.toLowerCase()); + const loaded = instance.getLoadedLanguages(); - if (isSpecialLang(name) || instance.getLoadedLanguages().includes(name)) { - return languageId; + // Shiki has bundled languages' names and aliases in lowercase + for (const name of [language, language.toLowerCase()]) { + if (isSpecialLang(name) || loaded.includes(name)) { + return name; + } } + const name = language.toLowerCase(); + if (!BUNDLED_NAMES.has(name)) { return FALLBACK_LANGUAGE; } instance.loadLanguageSync(importBundledLanguage(name)); - return languageId; + return name; }; /**