feat: add external Wasm build

README.md

@@ -67,6 +67,17 @@ const { exports, reexports } = parse(source);
 
 The Wasm build is around 1.5x faster and without a cold start.
 
+Node.js deployments can load Wasm from a separate file through the `cjs-module-lexer/external` entry:
+
+```js
+import { init, parse } from 'cjs-module-lexer/external';
+await init();
+const { exports, reexports } = parse(source);
+```
+
+This entry loads `lib/lexer.wasm` during initialization. It also supports `require('cjs-module-lexer/external')`
+with `initSync()`. The default entry remains self-contained.
+
 ### Grammar
 
 CommonJS exports matches are run against the source token stream.

build.js

@@ -1,25 +1,81 @@
+const assert = require('assert');
 const fs = require('fs');
-const terser = require('terser');
 
-const MINIFY = true;
+const babel = require('@babel/core');
+const terser = require('terser');
 
-try { fs.mkdirSync('./dist'); }
-catch (e) {}
+fs.mkdirSync('./dist', { recursive: true });
 
 const wasmBuffer = fs.readFileSync('./lib/lexer.wasm');
-const jsSource = fs.readFileSync('./src/lexer.js').toString();
-const pjson = JSON.parse(fs.readFileSync('./package.json').toString());
+const jsSource = fs.readFileSync('./src/lexer.js', 'utf8');
+const { version } = JSON.parse(fs.readFileSync('./package.json', 'utf8'));
+const preamble = `/* cjs-module-lexer ${version} */`;
+const inlineSource = jsSource.replace('WASM_BINARY', wasmBuffer.toString('base64'));
+assert.ok(!inlineSource.includes('WASM_BINARY'), 'Unresolved inline Wasm marker.');
+
+/**
+ * @param {string} imports
+ * @param {string} wasmBytes
+ */
+function renderExternal (imports, wasmBytes) {
+  const loaderStartMarker = 'function getWasmBytes() {';
+  const loaderStart = jsSource.indexOf(loaderStartMarker);
+  assert.notStrictEqual(loaderStart, -1, 'Missing getWasmBytes function.');
+  assert.strictEqual(
+    jsSource.indexOf(loaderStartMarker, loaderStart + loaderStartMarker.length),
+    -1,
+    'Multiple getWasmBytes functions.'
+  );
+  const loaderEnd = jsSource.indexOf('\nlet initPromise;', loaderStart);
+  assert.notStrictEqual(loaderEnd, -1, 'Missing getWasmBytes boundary.');
+  return `${imports}\n${jsSource.slice(0, loaderStart)}function getWasmBytes() {\n` +
+    `  return ${wasmBytes};\n}${jsSource.slice(loaderEnd)}`;
+}
 
-const jsSourceProcessed = jsSource.replace('WASM_BINARY', wasmBuffer.toString('base64'));
+/**
+ * @param {string} source
+ * @param {boolean} isModule
+ */
+function minify (source, isModule) {
+  const options = {
+    module: isModule,
+    output: { preamble }
+  };
+  const result = terser.minify(source, options);
+  assert.ifError(result.error);
+  return result.code;
+}
 
-const minified = MINIFY && terser.minify(jsSourceProcessed, {
-  module: true,
-  output: {
-    preamble: `/* cjs-module-lexer ${pjson.version} */`
-  }
-});
+/**
+ * @param {string} filename
+ * @param {string} source
+ */
+function buildEsm (filename, source) {
+  fs.writeFileSync(`./dist/${filename}.mjs`, minify(source, true));
+}
 
-if (minified.error)
-  throw minified.error;
+/**
+ * @param {string} filename
+ * @param {string} source
+ */
+function buildCjs (filename, source) {
+  const cjsModuleSource = minify(source, true);
+  const { code } = babel.transformSync(cjsModuleSource, { filename: `./dist/${filename}.mjs` });
+  fs.writeFileSync(`./dist/${filename}.js`, minify(code, false));
+}
 
-fs.writeFileSync('./dist/lexer.mjs', minified ? minified.code : jsSourceProcessed);
+buildEsm('lexer', inlineSource);
+buildEsm(
+  'lexer-external',
+  renderExternal(
+    `import { readFileSync } from 'fs';`,
+    `readFileSync(new URL('../lib/lexer.wasm', import.meta.url))`
+  )
+);
+buildCjs(
+  'lexer-external',
+  renderExternal(
+    `import { readFileSync } from 'fs';\nimport { join } from 'path';`,
+    `readFileSync(join(__dirname, '../lib/lexer.wasm'))`
+  )
+);

package.json

@@ -4,20 +4,38 @@
   "description": "Lexes CommonJS modules, returning their named exports metadata",
   "main": "lexer.js",
   "exports": {
-    "import": {
-      "types": "./lexer.d.mts",
-      "default": "./dist/lexer.mjs"
+    ".": {
+      "import": {
+        "types": "./lexer.d.mts",
+        "default": "./dist/lexer.mjs"
+      },
+      "default": "./lexer.js"
     },
-    "default": "./lexer.js"
+    "./external": {
+      "import": {
+        "types": "./lexer.d.mts",
+        "default": "./dist/lexer-external.mjs"
+      },
+      "default": {
+        "types": "./lexer.d.ts",
+        "default": "./dist/lexer-external.js"
+      }
+    }
   },
   "types": "lexer.d.ts",
   "scripts": {
-    "test-js": "cross-env NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js",
-    "test-wasm": "cross-env WASM=1 NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js",
-    "test-wasm-sync": "cross-env WASM_SYNC=1 NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js",
-    "test": "npm run test-wasm && npm run test-wasm-sync && npm run test-js",
+    "test-run": "cross-env NODE_OPTIONS=--disallow-code-generation-from-strings mocha -b -u tdd test/*.js",
+    "test-js": "npm run test-run",
+    "test-wasm": "cross-env WASM=1 npm run test-run",
+    "test-wasm-sync": "cross-env WASM_SYNC=1 npm run test-run",
+    "test-wasm-inline": "npm run test-wasm && npm run test-wasm-sync",
+    "test-wasm-external": "cross-env WASM_EXTERNAL=1 npm run test-run",
+    "test-wasm-external-sync": "cross-env WASM_EXTERNAL_SYNC=1 npm run test-run",
+    "test-wasm-external-all": "npm run test-wasm-external && npm run test-wasm-external-sync",
+    "test-package": "node test/package/index.js",
+    "test": "npm run test-wasm-inline && npm run test-wasm-external-all && npm run test-js && npm run test-package",
     "bench": "node --expose-gc bench/index.mjs",
-    "build": "node build.js ; babel dist/lexer.mjs -o dist/lexer.js ; terser dist/lexer.js -o dist/lexer.js",
+    "build": "node build.js && babel dist/lexer.mjs -o dist/lexer.js && terser dist/lexer.js -o dist/lexer.js",
     "build-wasm": "make lib/lexer.wasm ; node build.js",
     "prepublishOnly": "make && npm run build",
     "footprint": "npm run build && cat dist/lexer.js | gzip -9f | wc -c"
@@ -35,6 +53,7 @@
   },
   "files": [
     "dist",
+    "lib/lexer.wasm",
     "lexer.d.ts",
     "lexer.d.mts"
   ],

test/_unit.js

@@ -3,14 +3,22 @@ const assert = require('assert');
 let parse;
 async function loadParser () {
   if (parse) return;
-  if (process.env.WASM) {
-    const m = await import('../dist/lexer.mjs');
-    await m.init();
-    parse = m.parse;
+  if (process.env.WASM_EXTERNAL) {
+    const lexer = await import('cjs-module-lexer/external');
+    await lexer.init();
+    parse = lexer.parse;
+  } else if (process.env.WASM_EXTERNAL_SYNC) {
+    const lexer = require('cjs-module-lexer/external');
+    lexer.initSync();
+    parse = lexer.parse;
+  } else if (process.env.WASM) {
+    const lexer = await import('../dist/lexer.mjs');
+    await lexer.init();
+    parse = lexer.parse;
   } else if (process.env.WASM_SYNC) {
-    const m = require('../dist/lexer.js');
-    m.initSync();
-    parse = m.parse;
+    const lexer = require('../dist/lexer.js');
+    lexer.initSync();
+    parse = lexer.parse;
   }
   else {
     parse = require('../lexer.js').parse;

test/integration.js

@@ -4,14 +4,22 @@ const assert = require('assert');
 let parse;
 async function loadParser () {
   if (parse) return;
-  if (process.env.WASM) {
-    const m = await import('../dist/lexer.mjs');
-    await m.init();
-    parse = m.parse;
+  if (process.env.WASM_EXTERNAL) {
+    const lexer = await import('cjs-module-lexer/external');
+    await lexer.init();
+    parse = lexer.parse;
+  } else if (process.env.WASM_EXTERNAL_SYNC) {
+    const lexer = require('cjs-module-lexer/external');
+    lexer.initSync();
+    parse = lexer.parse;
+  } else if (process.env.WASM) {
+    const lexer = await import('../dist/lexer.mjs');
+    await lexer.init();
+    parse = lexer.parse;
   } else if (process.env.WASM_SYNC) {
-    const m = require('../dist/lexer.js');
-    m.initSync();
-    parse = m.parse;
+    const lexer = require('../dist/lexer.js');
+    lexer.initSync();
+    parse = lexer.parse;
   }
   else {
     parse = require('../lexer.js').parse;

test/package/index.js

@@ -0,0 +1,109 @@
+const assert = require('assert');
+const { execFileSync } = require('child_process');
+const {
+  copyFileSync,
+  mkdirSync,
+  mkdtempSync,
+  realpathSync,
+  rmSync,
+  writeFileSync
+} = require('fs');
+const { tmpdir } = require('os');
+const { join } = require('path');
+const { pathToFileURL } = require('url');
+
+/**
+ * @param {string} entryName
+ * @param {Buffer} [wasm]
+ */
+function createExternalFixture (entryName, wasm) {
+  const root = realpathSync(mkdtempSync(join(tmpdir(), 'cjs-module-lexer-external-')));
+  const entry = join(root, 'dist', entryName);
+  const wasmPath = join(root, 'lib', 'lexer.wasm');
+  mkdirSync(join(root, 'dist'), { recursive: true });
+  copyFileSync(join(__dirname, '..', '..', 'dist', entryName), entry);
+  if (wasm !== undefined) {
+    mkdirSync(join(root, 'lib'), { recursive: true });
+    writeFileSync(wasmPath, wasm);
+  }
+  return { entry, root, wasmPath };
+}
+
+/**
+ * @param {string} directory
+ */
+async function testExternalInitializers () {
+  let fixture = createExternalFixture('lexer-external.js');
+  try {
+    const lexer = require(fixture.entry);
+    assert.throws(() => lexer.initSync(), {
+      code: 'ENOENT',
+      path: fixture.wasmPath
+    });
+  } finally {
+    rmSync(fixture.root, { recursive: true, force: true });
+  }
+
+  fixture = createExternalFixture('lexer-external.mjs');
+  try {
+    const lexer = await import(pathToFileURL(fixture.entry).href);
+    await assert.rejects(lexer.init(), {
+      code: 'ENOENT',
+      path: fixture.wasmPath
+    });
+  } finally {
+    rmSync(fixture.root, { recursive: true, force: true });
+  }
+
+  fixture = createExternalFixture('lexer-external.js', Buffer.from('invalid Wasm'));
+  try {
+    const lexer = require(fixture.entry);
+    assert.throws(() => lexer.initSync(), WebAssembly.CompileError);
+  } finally {
+    rmSync(fixture.root, { recursive: true, force: true });
+  }
+
+  fixture = createExternalFixture('lexer-external.mjs', Buffer.from('invalid Wasm'));
+  try {
+    const lexer = await import(pathToFileURL(fixture.entry).href);
+    await assert.rejects(lexer.init(), WebAssembly.CompileError);
+  } finally {
+    rmSync(fixture.root, { recursive: true, force: true });
+  }
+}
+
+/**
+ * @param {Error} error
+ */
+function reportError (error) {
+  console.error(error);
+  process.exitCode = 1;
+}
+
+const npmCli = process.env.npm_execpath;
+assert.ok(npmCli, 'npm_execpath is required');
+
+const output = execFileSync(process.execPath, [
+  npmCli,
+  'pack',
+  '--dry-run',
+  '--json',
+  '--ignore-scripts'
+], {
+  encoding: 'utf8',
+  env: {
+    ...process.env,
+    npm_config_cache: join(tmpdir(), 'cjs-module-lexer-npm-cache')
+  }
+});
+const [{ files }] = JSON.parse(output);
+const packedFiles = new Set();
+for (const { path } of files) {
+  packedFiles.add(path);
+}
+
+assert.ok(packedFiles.has('dist/lexer-external.js'));
+assert.ok(packedFiles.has('dist/lexer-external.mjs'));
+assert.ok(packedFiles.has('lib/lexer.wasm'));
+
+testExternalInitializers().catch(reportError);