From b0b6fb379e25ac6de50190ced9e0c2923f1af6c5 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Sat, 29 Aug 2026 03:56:30 +0200 Subject: [PATCH 1/2] feat: add external Wasm build The opt-in Node.js entry reads the packaged Wasm asset during initialization, so importing the module does not access the file. The default entry remains self-contained. Fixes: https://github.com/nodejs/cjs-module-lexer/issues/96 --- README.md | 11 ++++ build.js | 88 ++++++++++++++++++++++++------ package.json | 37 +++++++++---- test/_unit.js | 22 +++++--- test/integration.js | 22 +++++--- test/package/index.js | 123 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 264 insertions(+), 39 deletions(-) create mode 100644 test/package/index.js diff --git a/README.md b/README.md index addf5ed..78c56fe 100755 --- a/README.md +++ b/README.md @@ -67,6 +67,17 @@ const { exports, reexports } = parse(source); The Wasm build is around 1.5x faster and without a cold start. +Node.js deployments can load Wasm from a separate file through the `cjs-module-lexer/external` entry: + +```js +import { init, parse } from 'cjs-module-lexer/external'; +await init(); +const { exports, reexports } = parse(source); +``` + +This entry loads `lib/lexer.wasm` during initialization. It also supports `require('cjs-module-lexer/external')` +with `initSync()`. The default entry remains self-contained. + ### Grammar CommonJS exports matches are run against the source token stream. diff --git a/build.js b/build.js index da024d1..8b52fd6 100755 --- a/build.js +++ b/build.js @@ -1,25 +1,81 @@ +const assert = require('assert'); const fs = require('fs'); -const terser = require('terser'); -const MINIFY = true; +const babel = require('@babel/core'); +const terser = require('terser'); -try { fs.mkdirSync('./dist'); } -catch (e) {} +fs.mkdirSync('./dist', { recursive: true }); const wasmBuffer = fs.readFileSync('./lib/lexer.wasm'); -const jsSource = fs.readFileSync('./src/lexer.js').toString(); -const pjson = JSON.parse(fs.readFileSync('./package.json').toString()); +const jsSource = fs.readFileSync('./src/lexer.js', 'utf8'); +const { version } = JSON.parse(fs.readFileSync('./package.json', 'utf8')); +const preamble = `/* cjs-module-lexer ${version} */`; +const inlineSource = jsSource.replace('WASM_BINARY', wasmBuffer.toString('base64')); +assert.ok(!inlineSource.includes('WASM_BINARY'), 'Unresolved inline Wasm marker.'); + +/** + * @param {string} imports + * @param {string} wasmBytes + */ +function renderExternal (imports, wasmBytes) { + const loaderStartMarker = 'function getWasmBytes() {'; + const loaderStart = jsSource.indexOf(loaderStartMarker); + assert.notStrictEqual(loaderStart, -1, 'Missing getWasmBytes function.'); + assert.strictEqual( + jsSource.indexOf(loaderStartMarker, loaderStart + loaderStartMarker.length), + -1, + 'Multiple getWasmBytes functions.' + ); + const loaderEnd = jsSource.indexOf('\nlet initPromise;', loaderStart); + assert.notStrictEqual(loaderEnd, -1, 'Missing getWasmBytes boundary.'); + return `${imports}\n${jsSource.slice(0, loaderStart)}function getWasmBytes() {\n` + + ` return ${wasmBytes};\n}${jsSource.slice(loaderEnd)}`; +} -const jsSourceProcessed = jsSource.replace('WASM_BINARY', wasmBuffer.toString('base64')); +/** + * @param {string} source + * @param {boolean} isModule + */ +function minify (source, isModule) { + const options = { + module: isModule, + output: { preamble } + }; + const result = terser.minify(source, options); + assert.ifError(result.error); + return result.code; +} -const minified = MINIFY && terser.minify(jsSourceProcessed, { - module: true, - output: { - preamble: `/* cjs-module-lexer ${pjson.version} */` - } -}); +/** + * @param {string} filename + * @param {string} source + */ +function buildEsm (filename, source) { + fs.writeFileSync(`./dist/${filename}.mjs`, minify(source, true)); +} -if (minified.error) - throw minified.error; +/** + * @param {string} filename + * @param {string} source + */ +function buildCjs (filename, source) { + const cjsModuleSource = minify(source, true); + const { code } = babel.transformSync(cjsModuleSource, { filename: `./dist/${filename}.mjs` }); + fs.writeFileSync(`./dist/${filename}.js`, minify(code, false)); +} -fs.writeFileSync('./dist/lexer.mjs', minified ? minified.code : jsSourceProcessed); +buildEsm('lexer', inlineSource); +buildEsm( + 'lexer-external', + renderExternal( + `import { readFileSync } from 'fs';`, + `readFileSync(new URL('../lib/lexer.wasm', import.meta.url))` + ) +); +buildCjs( + 'lexer-external', + renderExternal( + `import { readFileSync } from 'fs';\nimport { join } from 'path';`, + `readFileSync(join(__dirname, '../lib/lexer.wasm'))` + ) +); diff --git a/package.json b/package.json index c1edff5..64265a3 100755 --- a/package.json +++ b/package.json @@ -4,20 +4,38 @@ "description": "Lexes CommonJS modules, returning their named exports metadata", "main": "lexer.js", "exports": { - "import": { - "types": "./lexer.d.mts", - "default": "./dist/lexer.mjs" + ".": { + "import": { + "types": "./lexer.d.mts", + "default": "./dist/lexer.mjs" + }, + "default": "./lexer.js" }, - "default": "./lexer.js" + "./external": { + "import": { + "types": "./lexer.d.mts", + "default": "./dist/lexer-external.mjs" + }, + "default": { + "types": "./lexer.d.ts", + "default": "./dist/lexer-external.js" + } + } }, "types": "lexer.d.ts", "scripts": { - "test-js": "cross-env NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js", - "test-wasm": "cross-env WASM=1 NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js", - "test-wasm-sync": "cross-env WASM_SYNC=1 NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js", - "test": "npm run test-wasm && npm run test-wasm-sync && npm run test-js", + "test-run": "cross-env NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js", + "test-js": "npm run test-run", + "test-wasm": "cross-env WASM=1 npm run test-run", + "test-wasm-sync": "cross-env WASM_SYNC=1 npm run test-run", + "test-wasm-inline": "npm run test-wasm && npm run test-wasm-sync", + "test-wasm-external": "cross-env WASM_EXTERNAL=1 npm run test-run", + "test-wasm-external-sync": "cross-env WASM_EXTERNAL_SYNC=1 npm run test-run", + "test-wasm-external-all": "npm run test-wasm-external && npm run test-wasm-external-sync", + "test-package": "node test/package/index.js", + "test": "npm run test-wasm-inline && npm run test-wasm-external-all && npm run test-js && npm run test-package", "bench": "node --expose-gc bench/index.mjs", - "build": "node build.js ; babel dist/lexer.mjs -o dist/lexer.js ; terser dist/lexer.js -o dist/lexer.js", + "build": "node build.js && babel dist/lexer.mjs -o dist/lexer.js && terser dist/lexer.js -o dist/lexer.js", "build-wasm": "make lib/lexer.wasm ; node build.js", "prepublishOnly": "make && npm run build", "footprint": "npm run build && cat dist/lexer.js | gzip -9f | wc -c" @@ -35,6 +53,7 @@ }, "files": [ "dist", + "lib/lexer.wasm", "lexer.d.ts", "lexer.d.mts" ], diff --git a/test/_unit.js b/test/_unit.js index 0458f17..9f8ddbf 100755 --- a/test/_unit.js +++ b/test/_unit.js @@ -3,14 +3,22 @@ const assert = require('assert'); let parse; async function loadParser () { if (parse) return; - if (process.env.WASM) { - const m = await import('../dist/lexer.mjs'); - await m.init(); - parse = m.parse; + if (process.env.WASM_EXTERNAL) { + const lexer = await import('cjs-module-lexer/external'); + await lexer.init(); + parse = lexer.parse; + } else if (process.env.WASM_EXTERNAL_SYNC) { + const lexer = require('cjs-module-lexer/external'); + lexer.initSync(); + parse = lexer.parse; + } else if (process.env.WASM) { + const lexer = await import('../dist/lexer.mjs'); + await lexer.init(); + parse = lexer.parse; } else if (process.env.WASM_SYNC) { - const m = require('../dist/lexer.js'); - m.initSync(); - parse = m.parse; + const lexer = require('../dist/lexer.js'); + lexer.initSync(); + parse = lexer.parse; } else { parse = require('../lexer.js').parse; diff --git a/test/integration.js b/test/integration.js index 1a99de9..a6c7bb1 100755 --- a/test/integration.js +++ b/test/integration.js @@ -4,14 +4,22 @@ const assert = require('assert'); let parse; async function loadParser () { if (parse) return; - if (process.env.WASM) { - const m = await import('../dist/lexer.mjs'); - await m.init(); - parse = m.parse; + if (process.env.WASM_EXTERNAL) { + const lexer = await import('cjs-module-lexer/external'); + await lexer.init(); + parse = lexer.parse; + } else if (process.env.WASM_EXTERNAL_SYNC) { + const lexer = require('cjs-module-lexer/external'); + lexer.initSync(); + parse = lexer.parse; + } else if (process.env.WASM) { + const lexer = await import('../dist/lexer.mjs'); + await lexer.init(); + parse = lexer.parse; } else if (process.env.WASM_SYNC) { - const m = require('../dist/lexer.js'); - m.initSync(); - parse = m.parse; + const lexer = require('../dist/lexer.js'); + lexer.initSync(); + parse = lexer.parse; } else { parse = require('../lexer.js').parse; diff --git a/test/package/index.js b/test/package/index.js new file mode 100644 index 0000000..55e343e --- /dev/null +++ b/test/package/index.js @@ -0,0 +1,123 @@ +const assert = require('assert'); +const { execFileSync } = require('child_process'); +const { + copyFileSync, + mkdirSync, + mkdtempSync, + realpathSync, + readdirSync, + rmdirSync, + unlinkSync, + writeFileSync +} = require('fs'); +const { tmpdir } = require('os'); +const { join } = require('path'); +const { pathToFileURL } = require('url'); + +/** + * @param {string} entryName + * @param {Buffer} [wasm] + */ +function createExternalFixture (entryName, wasm) { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'cjs-module-lexer-external-'))); + const entry = join(root, 'dist', entryName); + const wasmPath = join(root, 'lib', 'lexer.wasm'); + mkdirSync(join(root, 'dist'), { recursive: true }); + copyFileSync(join(__dirname, '..', '..', 'dist', entryName), entry); + if (wasm !== undefined) { + mkdirSync(join(root, 'lib'), { recursive: true }); + writeFileSync(wasmPath, wasm); + } + return { entry, root, wasmPath }; +} + +/** + * @param {string} directory + */ +function removeDirectory (directory) { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + removeDirectory(path); + } else { + unlinkSync(path); + } + } + rmdirSync(directory); +} + +async function testExternalInitializers () { + let fixture = createExternalFixture('lexer-external.js'); + try { + const lexer = require(fixture.entry); + assert.throws(() => lexer.initSync(), { + code: 'ENOENT', + path: fixture.wasmPath + }); + } finally { + removeDirectory(fixture.root); + } + + fixture = createExternalFixture('lexer-external.mjs'); + try { + const lexer = await import(pathToFileURL(fixture.entry).href); + await assert.rejects(lexer.init(), { + code: 'ENOENT', + path: fixture.wasmPath + }); + } finally { + removeDirectory(fixture.root); + } + + fixture = createExternalFixture('lexer-external.js', Buffer.from('invalid Wasm')); + try { + const lexer = require(fixture.entry); + assert.throws(() => lexer.initSync(), WebAssembly.CompileError); + } finally { + removeDirectory(fixture.root); + } + + fixture = createExternalFixture('lexer-external.mjs', Buffer.from('invalid Wasm')); + try { + const lexer = await import(pathToFileURL(fixture.entry).href); + await assert.rejects(lexer.init(), WebAssembly.CompileError); + } finally { + removeDirectory(fixture.root); + } +} + +/** + * @param {Error} error + */ +function reportError (error) { + console.error(error); + process.exitCode = 1; +} + +const npmCli = process.env.npm_execpath; +assert.ok(npmCli, 'npm_execpath is required'); + +const output = execFileSync(process.execPath, [ + npmCli, + 'pack', + '--dry-run', + '--json', + '--ignore-scripts' +], { + encoding: 'utf8', + env: { + ...process.env, + npm_config_cache: join(tmpdir(), 'cjs-module-lexer-npm-cache') + } +}); +const [{ files }] = JSON.parse(output); +const packedFiles = new Set(); +for (const { path } of files) { + packedFiles.add(path); +} + +assert.ok(packedFiles.has('dist/lexer-external.js')); +assert.ok(packedFiles.has('dist/lexer-external.mjs')); +assert.ok(packedFiles.has('lib/lexer.wasm')); + +testExternalInitializers().catch(reportError); From 7047ba8081c364505d2eafb54b80fcff6e0c41b8 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Mon, 31 Aug 2026 12:50:04 +0200 Subject: [PATCH 2/2] test: simplify external Wasm fixture cleanup --- test/package/index.js | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/test/package/index.js b/test/package/index.js index 55e343e..8052b7d 100644 --- a/test/package/index.js +++ b/test/package/index.js @@ -5,9 +5,7 @@ const { mkdirSync, mkdtempSync, realpathSync, - readdirSync, - rmdirSync, - unlinkSync, + rmSync, writeFileSync } = require('fs'); const { tmpdir } = require('os'); @@ -34,18 +32,6 @@ function createExternalFixture (entryName, wasm) { /** * @param {string} directory */ -function removeDirectory (directory) { - for (const entry of readdirSync(directory, { withFileTypes: true })) { - const path = join(directory, entry.name); - if (entry.isDirectory()) { - removeDirectory(path); - } else { - unlinkSync(path); - } - } - rmdirSync(directory); -} - async function testExternalInitializers () { let fixture = createExternalFixture('lexer-external.js'); try { @@ -55,7 +41,7 @@ async function testExternalInitializers () { path: fixture.wasmPath }); } finally { - removeDirectory(fixture.root); + rmSync(fixture.root, { recursive: true, force: true }); } fixture = createExternalFixture('lexer-external.mjs'); @@ -66,7 +52,7 @@ async function testExternalInitializers () { path: fixture.wasmPath }); } finally { - removeDirectory(fixture.root); + rmSync(fixture.root, { recursive: true, force: true }); } fixture = createExternalFixture('lexer-external.js', Buffer.from('invalid Wasm')); @@ -74,7 +60,7 @@ async function testExternalInitializers () { const lexer = require(fixture.entry); assert.throws(() => lexer.initSync(), WebAssembly.CompileError); } finally { - removeDirectory(fixture.root); + rmSync(fixture.root, { recursive: true, force: true }); } fixture = createExternalFixture('lexer-external.mjs', Buffer.from('invalid Wasm')); @@ -82,7 +68,7 @@ async function testExternalInitializers () { const lexer = await import(pathToFileURL(fixture.entry).href); await assert.rejects(lexer.init(), WebAssembly.CompileError); } finally { - removeDirectory(fixture.root); + rmSync(fixture.root, { recursive: true, force: true }); } }