diff --git a/README.md b/README.md index addf5ed..78c56fe 100755 --- a/README.md +++ b/README.md @@ -67,6 +67,17 @@ const { exports, reexports } = parse(source); The Wasm build is around 1.5x faster and without a cold start. +Node.js deployments can load Wasm from a separate file through the `cjs-module-lexer/external` entry: + +```js +import { init, parse } from 'cjs-module-lexer/external'; +await init(); +const { exports, reexports } = parse(source); +``` + +This entry loads `lib/lexer.wasm` during initialization. It also supports `require('cjs-module-lexer/external')` +with `initSync()`. The default entry remains self-contained. + ### Grammar CommonJS exports matches are run against the source token stream. diff --git a/build.js b/build.js index da024d1..8b52fd6 100755 --- a/build.js +++ b/build.js @@ -1,25 +1,81 @@ +const assert = require('assert'); const fs = require('fs'); -const terser = require('terser'); -const MINIFY = true; +const babel = require('@babel/core'); +const terser = require('terser'); -try { fs.mkdirSync('./dist'); } -catch (e) {} +fs.mkdirSync('./dist', { recursive: true }); const wasmBuffer = fs.readFileSync('./lib/lexer.wasm'); -const jsSource = fs.readFileSync('./src/lexer.js').toString(); -const pjson = JSON.parse(fs.readFileSync('./package.json').toString()); +const jsSource = fs.readFileSync('./src/lexer.js', 'utf8'); +const { version } = JSON.parse(fs.readFileSync('./package.json', 'utf8')); +const preamble = `/* cjs-module-lexer ${version} */`; +const inlineSource = jsSource.replace('WASM_BINARY', wasmBuffer.toString('base64')); +assert.ok(!inlineSource.includes('WASM_BINARY'), 'Unresolved inline Wasm marker.'); + +/** + * @param {string} imports + * @param {string} wasmBytes + */ +function renderExternal (imports, wasmBytes) { + const loaderStartMarker = 'function getWasmBytes() {'; + const loaderStart = jsSource.indexOf(loaderStartMarker); + assert.notStrictEqual(loaderStart, -1, 'Missing getWasmBytes function.'); + assert.strictEqual( + jsSource.indexOf(loaderStartMarker, loaderStart + loaderStartMarker.length), + -1, + 'Multiple getWasmBytes functions.' + ); + const loaderEnd = jsSource.indexOf('\nlet initPromise;', loaderStart); + assert.notStrictEqual(loaderEnd, -1, 'Missing getWasmBytes boundary.'); + return `${imports}\n${jsSource.slice(0, loaderStart)}function getWasmBytes() {\n` + + ` return ${wasmBytes};\n}${jsSource.slice(loaderEnd)}`; +} -const jsSourceProcessed = jsSource.replace('WASM_BINARY', wasmBuffer.toString('base64')); +/** + * @param {string} source + * @param {boolean} isModule + */ +function minify (source, isModule) { + const options = { + module: isModule, + output: { preamble } + }; + const result = terser.minify(source, options); + assert.ifError(result.error); + return result.code; +} -const minified = MINIFY && terser.minify(jsSourceProcessed, { - module: true, - output: { - preamble: `/* cjs-module-lexer ${pjson.version} */` - } -}); +/** + * @param {string} filename + * @param {string} source + */ +function buildEsm (filename, source) { + fs.writeFileSync(`./dist/${filename}.mjs`, minify(source, true)); +} -if (minified.error) - throw minified.error; +/** + * @param {string} filename + * @param {string} source + */ +function buildCjs (filename, source) { + const cjsModuleSource = minify(source, true); + const { code } = babel.transformSync(cjsModuleSource, { filename: `./dist/${filename}.mjs` }); + fs.writeFileSync(`./dist/${filename}.js`, minify(code, false)); +} -fs.writeFileSync('./dist/lexer.mjs', minified ? minified.code : jsSourceProcessed); +buildEsm('lexer', inlineSource); +buildEsm( + 'lexer-external', + renderExternal( + `import { readFileSync } from 'fs';`, + `readFileSync(new URL('../lib/lexer.wasm', import.meta.url))` + ) +); +buildCjs( + 'lexer-external', + renderExternal( + `import { readFileSync } from 'fs';\nimport { join } from 'path';`, + `readFileSync(join(__dirname, '../lib/lexer.wasm'))` + ) +); diff --git a/package.json b/package.json index c1edff5..64265a3 100755 --- a/package.json +++ b/package.json @@ -4,20 +4,38 @@ "description": "Lexes CommonJS modules, returning their named exports metadata", "main": "lexer.js", "exports": { - "import": { - "types": "./lexer.d.mts", - "default": "./dist/lexer.mjs" + ".": { + "import": { + "types": "./lexer.d.mts", + "default": "./dist/lexer.mjs" + }, + "default": "./lexer.js" }, - "default": "./lexer.js" + "./external": { + "import": { + "types": "./lexer.d.mts", + "default": "./dist/lexer-external.mjs" + }, + "default": { + "types": "./lexer.d.ts", + "default": "./dist/lexer-external.js" + } + } }, "types": "lexer.d.ts", "scripts": { - "test-js": "cross-env NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js", - "test-wasm": "cross-env WASM=1 NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js", - "test-wasm-sync": "cross-env WASM_SYNC=1 NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js", - "test": "npm run test-wasm && npm run test-wasm-sync && npm run test-js", + "test-run": "cross-env NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js", + "test-js": "npm run test-run", + "test-wasm": "cross-env WASM=1 npm run test-run", + "test-wasm-sync": "cross-env WASM_SYNC=1 npm run test-run", + "test-wasm-inline": "npm run test-wasm && npm run test-wasm-sync", + "test-wasm-external": "cross-env WASM_EXTERNAL=1 npm run test-run", + "test-wasm-external-sync": "cross-env WASM_EXTERNAL_SYNC=1 npm run test-run", + "test-wasm-external-all": "npm run test-wasm-external && npm run test-wasm-external-sync", + "test-package": "node test/package/index.js", + "test": "npm run test-wasm-inline && npm run test-wasm-external-all && npm run test-js && npm run test-package", "bench": "node --expose-gc bench/index.mjs", - "build": "node build.js ; babel dist/lexer.mjs -o dist/lexer.js ; terser dist/lexer.js -o dist/lexer.js", + "build": "node build.js && babel dist/lexer.mjs -o dist/lexer.js && terser dist/lexer.js -o dist/lexer.js", "build-wasm": "make lib/lexer.wasm ; node build.js", "prepublishOnly": "make && npm run build", "footprint": "npm run build && cat dist/lexer.js | gzip -9f | wc -c" @@ -35,6 +53,7 @@ }, "files": [ "dist", + "lib/lexer.wasm", "lexer.d.ts", "lexer.d.mts" ], diff --git a/test/_unit.js b/test/_unit.js index 0458f17..9f8ddbf 100755 --- a/test/_unit.js +++ b/test/_unit.js @@ -3,14 +3,22 @@ const assert = require('assert'); let parse; async function loadParser () { if (parse) return; - if (process.env.WASM) { - const m = await import('../dist/lexer.mjs'); - await m.init(); - parse = m.parse; + if (process.env.WASM_EXTERNAL) { + const lexer = await import('cjs-module-lexer/external'); + await lexer.init(); + parse = lexer.parse; + } else if (process.env.WASM_EXTERNAL_SYNC) { + const lexer = require('cjs-module-lexer/external'); + lexer.initSync(); + parse = lexer.parse; + } else if (process.env.WASM) { + const lexer = await import('../dist/lexer.mjs'); + await lexer.init(); + parse = lexer.parse; } else if (process.env.WASM_SYNC) { - const m = require('../dist/lexer.js'); - m.initSync(); - parse = m.parse; + const lexer = require('../dist/lexer.js'); + lexer.initSync(); + parse = lexer.parse; } else { parse = require('../lexer.js').parse; diff --git a/test/integration.js b/test/integration.js index 1a99de9..a6c7bb1 100755 --- a/test/integration.js +++ b/test/integration.js @@ -4,14 +4,22 @@ const assert = require('assert'); let parse; async function loadParser () { if (parse) return; - if (process.env.WASM) { - const m = await import('../dist/lexer.mjs'); - await m.init(); - parse = m.parse; + if (process.env.WASM_EXTERNAL) { + const lexer = await import('cjs-module-lexer/external'); + await lexer.init(); + parse = lexer.parse; + } else if (process.env.WASM_EXTERNAL_SYNC) { + const lexer = require('cjs-module-lexer/external'); + lexer.initSync(); + parse = lexer.parse; + } else if (process.env.WASM) { + const lexer = await import('../dist/lexer.mjs'); + await lexer.init(); + parse = lexer.parse; } else if (process.env.WASM_SYNC) { - const m = require('../dist/lexer.js'); - m.initSync(); - parse = m.parse; + const lexer = require('../dist/lexer.js'); + lexer.initSync(); + parse = lexer.parse; } else { parse = require('../lexer.js').parse; diff --git a/test/package/index.js b/test/package/index.js new file mode 100644 index 0000000..8052b7d --- /dev/null +++ b/test/package/index.js @@ -0,0 +1,109 @@ +const assert = require('assert'); +const { execFileSync } = require('child_process'); +const { + copyFileSync, + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + writeFileSync +} = require('fs'); +const { tmpdir } = require('os'); +const { join } = require('path'); +const { pathToFileURL } = require('url'); + +/** + * @param {string} entryName + * @param {Buffer} [wasm] + */ +function createExternalFixture (entryName, wasm) { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'cjs-module-lexer-external-'))); + const entry = join(root, 'dist', entryName); + const wasmPath = join(root, 'lib', 'lexer.wasm'); + mkdirSync(join(root, 'dist'), { recursive: true }); + copyFileSync(join(__dirname, '..', '..', 'dist', entryName), entry); + if (wasm !== undefined) { + mkdirSync(join(root, 'lib'), { recursive: true }); + writeFileSync(wasmPath, wasm); + } + return { entry, root, wasmPath }; +} + +/** + * @param {string} directory + */ +async function testExternalInitializers () { + let fixture = createExternalFixture('lexer-external.js'); + try { + const lexer = require(fixture.entry); + assert.throws(() => lexer.initSync(), { + code: 'ENOENT', + path: fixture.wasmPath + }); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } + + fixture = createExternalFixture('lexer-external.mjs'); + try { + const lexer = await import(pathToFileURL(fixture.entry).href); + await assert.rejects(lexer.init(), { + code: 'ENOENT', + path: fixture.wasmPath + }); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } + + fixture = createExternalFixture('lexer-external.js', Buffer.from('invalid Wasm')); + try { + const lexer = require(fixture.entry); + assert.throws(() => lexer.initSync(), WebAssembly.CompileError); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } + + fixture = createExternalFixture('lexer-external.mjs', Buffer.from('invalid Wasm')); + try { + const lexer = await import(pathToFileURL(fixture.entry).href); + await assert.rejects(lexer.init(), WebAssembly.CompileError); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } +} + +/** + * @param {Error} error + */ +function reportError (error) { + console.error(error); + process.exitCode = 1; +} + +const npmCli = process.env.npm_execpath; +assert.ok(npmCli, 'npm_execpath is required'); + +const output = execFileSync(process.execPath, [ + npmCli, + 'pack', + '--dry-run', + '--json', + '--ignore-scripts' +], { + encoding: 'utf8', + env: { + ...process.env, + npm_config_cache: join(tmpdir(), 'cjs-module-lexer-npm-cache') + } +}); +const [{ files }] = JSON.parse(output); +const packedFiles = new Set(); +for (const { path } of files) { + packedFiles.add(path); +} + +assert.ok(packedFiles.has('dist/lexer-external.js')); +assert.ok(packedFiles.has('dist/lexer-external.mjs')); +assert.ok(packedFiles.has('lib/lexer.wasm')); + +testExternalInitializers().catch(reportError);