From 4d5f108fe2b7ecf11668d4033c770350e5d4f4f0 Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Mon, 26 Oct 2026 09:48:42 +0000 Subject: [PATCH 1/9] fix: export snapshots using the SQLite backup API --- .github/workflows/CI.yml | 8 ++ src/api.js | 93 +++++++++++--- src/exported_functions.json | 3 + test/compare_exports.cjs | 189 +++++++++++++++++++++++++++++ test/test_export.js | 186 ++++++++++++++++++++++++++++ test/test_function_registration.js | 2 +- 6 files changed, 466 insertions(+), 15 deletions(-) create mode 100644 test/compare_exports.cjs create mode 100644 test/test_export.js diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a272fe5e..3b45b9e8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,6 +4,7 @@ on: [push, pull_request] permissions: contents: read + actions: read jobs: build: @@ -24,6 +25,13 @@ jobs: if-no-files-found: error - name: test WebAssembly builds and distribution behavior run: npm ci && npm test + # Temporary experiment for the competing export PRs; remove after recording results. + - name: Compare backup export with the tested serialize PR + env: + GH_TOKEN: ${{ github.token }} + run: | + gh run download 40301139618 --repo sql-js/sql.js --name dist --dir "$RUNNER_TEMP/serialize-dist" + node test/compare_exports.cjs "$RUNNER_TEMP/serialize-dist" dist - name: generate documentation run: npm run doc - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/src/api.js b/src/api.js index 864607e5..810e22c1 100644 --- a/src/api.js +++ b/src/api.js @@ -60,6 +60,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var NULL = 0; // SQLite enum var SQLITE_OK = 0; + var SQLITE_BUSY = 5; + var SQLITE_LOCKED = 6; var SQLITE_ROW = 100; var SQLITE_DONE = 101; var SQLITE_INTEGER = 1; @@ -75,6 +77,17 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // var - cwrap function var sqlite3_open = cwrap("sqlite3_open", "number", ["string", "number"]); var sqlite3_close_v2 = cwrap("sqlite3_close_v2", "number", ["number"]); + var sqlite3_backup_init = cwrap( + "sqlite3_backup_init", + "number", + ["number", "string", "number", "string"] + ); + var sqlite3_backup_step = cwrap( + "sqlite3_backup_step", "number", ["number", "number"] + ); + var sqlite3_backup_finish = cwrap( + "sqlite3_backup_finish", "number", ["number"] + ); var sqlite3_exec = cwrap( "sqlite3_exec", "number", @@ -1103,23 +1116,75 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { return new StatementIterator(sql, this); }; - /** Exports the contents of the database to a binary array. This - * operation will close and re-open the database which will cause - * any pragmas to be set back to their default values. + /** Exports a snapshot of the main database to a binary array without + * closing the connection. Prepared statements, registered functions, + * update hooks and connection settings (including pragmas) are preserved. + * TEMP objects and attached databases remain available on this connection, + * but are not included in the exported file. + * + * Exporting during an active write transaction throws an error. Commit or + * roll back the transaction before exporting. Read transactions are + * allowed. Exporting never commits or rolls back a transaction or savepoint, + * including when the export fails. + * + * The exported file uses rollback-journal format, even if this connection + * uses WAL mode. + * + * The returned array is an independent copy. It can be used to create a + * new Database, but JavaScript functions, hooks and connection settings + * are not stored in the file and must be configured on the new connection. @return {Uint8Array} An array of bytes of the SQLite3 database file */ Database.prototype.export = function exportDatabase() { - Object.values(this.statements).forEach(function each(stmt) { - stmt.free(); - }); - this.functions.forEach(removeFunctionPointers); - this.functions.clear(); - this.handleError(sqlite3_close_v2(this.db)); - var binaryDb = FS.readFile(this.filename, { encoding: "binary" }); - this.handleError(sqlite3_open(this.filename, apiTemp)); - this.db = getValue(apiTemp, "i32"); - registerExtensionFunctions(this.db); - return binaryDb; + if (!this.db) { + throw new Error("Database closed"); + } + var filename = this.filename + "_export"; + FS.createDataFile("/", filename, [], true, true); + var destination = NULL; + var backup = NULL; + try { + var rc = sqlite3_open(filename, apiTemp); + destination = getValue(apiTemp, "i32"); + if (rc !== SQLITE_OK) { + throw new Error(sqlite3_errmsg(destination)); + } + backup = sqlite3_backup_init(destination, "main", this.db, "main"); + if (backup === NULL) { + throw new Error(sqlite3_errmsg(destination)); + } + rc = sqlite3_backup_step(backup, -1); + // Do not retry: this synchronous call cannot let a writer progress. + if (rc === SQLITE_BUSY || rc === SQLITE_LOCKED) { + throw new Error("Unable to export: SQLite is busy or locked. " + + "Finish active write transactions before exporting."); + } + if (rc !== SQLITE_DONE) { + throw new Error(sqlite3_errmsg(destination)); + } + rc = sqlite3_backup_finish(backup); + backup = NULL; + if (rc !== SQLITE_OK) { + throw new Error(sqlite3_errmsg(destination)); + } + rc = sqlite3_close_v2(destination); + if (rc !== SQLITE_OK) { + throw new Error(sqlite3_errmsg(destination)); + } + destination = NULL; + var binaryDb = FS.readFile(filename, { encoding: "binary" }); + if (binaryDb.length > 0) { + // All pages, including WAL pages, are in the backup. + // Make the exported copy independent of WAL support. + binaryDb[18] = 1; + binaryDb[19] = 1; + } + return binaryDb; + } finally { + if (backup !== NULL) sqlite3_backup_finish(backup); + if (destination !== NULL) sqlite3_close_v2(destination); + FS.unlink("/" + filename); + } }; /** Close the database, and all associated prepared statements. diff --git a/src/exported_functions.json b/src/exported_functions.json index 3be25955..d61b648a 100644 --- a/src/exported_functions.json +++ b/src/exported_functions.json @@ -2,6 +2,9 @@ "_malloc", "_free", "_sqlite3_open", +"_sqlite3_backup_init", +"_sqlite3_backup_step", +"_sqlite3_backup_finish", "_sqlite3_exec", "_sqlite3_free", "_sqlite3_errmsg", diff --git a/test/compare_exports.cjs b/test/compare_exports.cjs new file mode 100644 index 00000000..d3801e7b --- /dev/null +++ b/test/compare_exports.cjs @@ -0,0 +1,189 @@ +// CI-only comparison: node test/compare_exports.cjs SERIALIZE_DIST BACKUP_DIST +// Each measurement uses a fresh child process; both implementations run on +// the same machine. Timing is informational, never a pass/fail threshold. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { performance } = require('node:perf_hooks'); +const { gzipSync } = require('node:zlib'); + +function values(db, query) { + return db.exec(query)[0].values; +} + +async function worker(strategy, directory, mode, rows) { + const SQL = await require(path.resolve(directory, 'sql-wasm.js'))(); + const db = new SQL.Database(); + try { + if (mode === 'behavior') { + const outcomes = {}; + db.run('CREATE TABLE t(x); INSERT INTO t VALUES (1)'); + for (const [name, begin, expected] of [ + ['read_transaction', 'BEGIN; SELECT * FROM t', [[1]]], + ['reserved_write_transaction', 'BEGIN IMMEDIATE', [[1]]], + ['uncommitted_writes', 'BEGIN; INSERT INTO t VALUES (2)', [[1], [2]]] + ]) { + db.run(begin); + let bytes; + let error; + try { bytes = db.export(); } catch (e) { error = e; } + if (strategy === 'backup' && name !== 'read_transaction') { + assert.match(error && error.message, /busy or locked/); + outcomes[name] = 'rejected; source transaction intact'; + } else { + assert.ifError(error); + const copy = new SQL.Database(bytes); + try { + assert.deepEqual(values(copy, 'SELECT x FROM t ORDER BY x'), expected); + } finally { + copy.close(); + } + outcomes[name] = 'exported; source transaction intact'; + } + assert.deepEqual(values(db, 'SELECT x FROM t ORDER BY x'), expected); + db.run('ROLLBACK'); + assert.deepEqual(values(db, 'SELECT x FROM t'), [[1]]); + } + db.run('PRAGMA query_only = ON'); + const copy = new SQL.Database(db.export()); + try { + assert.deepEqual(values(copy, 'SELECT x FROM t'), [[1]]); + } finally { + copy.close(); + } + assert.deepEqual(values(db, 'PRAGMA query_only'), [[1]]); + outcomes.query_only = 'exported; setting preserved'; + return outcomes; + } + + db.run('CREATE TABLE payload(id INTEGER PRIMARY KEY, data BLOB);' + + 'WITH RECURSIVE seq(n) AS (VALUES(1) UNION ALL ' + + 'SELECT n+1 FROM seq WHERE n < ' + rows + ') ' + + 'INSERT INTO payload SELECT n, zeroblob(4096) FROM seq;' + + 'CREATE INDEX payload_length ON payload(length(data));'); + global.gc(); + const beforeRss = process.memoryUsage().rss; + const beforePeak = process.resourceUsage().maxRSS * 1024; + let start = performance.now(); + let bytes = db.export(); + const firstMs = performance.now() - start; + const outputBytes = bytes.length; + const firstRss = process.memoryUsage().rss; + const samplesMs = []; + for (let i = 0; i < 7; i++) { + bytes = null; + global.gc(); + start = performance.now(); + bytes = db.export(); + samplesMs.push(performance.now() - start); + assert.equal(bytes.length, outputBytes); + } + // Capture export-only memory before opening the independent copy. + const peakRss = process.resourceUsage().maxRSS * 1024; + const copy = new SQL.Database(bytes); + try { + assert.deepEqual(values(copy, 'PRAGMA integrity_check'), [['ok']]); + assert.deepEqual(values(copy, + 'SELECT count(*), sum(length(data)), min(id), max(id) FROM payload'), + [[rows, rows * 4096, 1, rows]]); + } finally { + copy.close(); + } + return { outputBytes, firstMs, samplesMs, beforeRss, beforePeak, firstRss, peakRss }; + } finally { + db.close(); + } +} + +function median(numbers) { + const sorted = numbers.slice().sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +} + +function child(strategy, directory, mode, rows = 0) { + const result = spawnSync(process.execPath, [ + '--expose-gc', __filename, '--worker', strategy, directory, mode, String(rows) + ], { encoding: 'utf8', timeout: 120000, maxBuffer: 1024 * 1024 }); + if (result.error) throw result.error; + assert.equal(result.status, 0, result.stderr || result.stdout); + return JSON.parse(result.stdout); +} + +function compare() { + const directories = { + serialize: path.resolve(process.argv[2]), + backup: path.resolve(process.argv[3]) + }; + const behavior = {}; + const bundle = {}; + for (const strategy of Object.keys(directories)) { + behavior[strategy] = child(strategy, directories[strategy], 'behavior'); + bundle[strategy] = {}; + for (const file of ['sql-wasm.js', 'sql-wasm.wasm']) { + const bytes = fs.readFileSync(path.join(directories[strategy], file)); + bundle[strategy][file] = { bytes: bytes.length, gzipBytes: gzipSync(bytes).length }; + } + } + const measurements = []; + for (const rows of [256, 4096, 16384]) { + const trials = { serialize: [], backup: [] }; + for (let repeat = 0; repeat < 3; repeat++) { + const order = repeat % 2 ? ['backup', 'serialize'] : ['serialize', 'backup']; + for (const strategy of order) { + trials[strategy].push(child(strategy, directories[strategy], 'timing', rows)); + } + } + for (const strategy of Object.keys(trials)) { + const runs = trials[strategy]; + measurements.push({ + strategy, + rows, + outputMiB: runs[0].outputBytes / 1048576, + firstMs: median(runs.map(run => run.firstMs)), + steadyMs: median(runs.flatMap(run => run.samplesMs)), + beforeRssMiB: median(runs.map(run => run.beforeRss / 1048576)), + firstRssMiB: median(runs.map(run => run.firstRss / 1048576)), + peakRssMiB: median(runs.map(run => run.peakRss / 1048576)), + runs + }); + } + } + const report = { + environment: { node: process.version, cpu: os.cpus()[0].model, platform: os.platform() }, + behavior, bundle, measurements + }; + console.log(JSON.stringify(report, null, 2)); + const lines = [ + '## Export implementation comparison', + '', + 'Same runner, optimized wasm, 3 fresh processes per strategy/size.', + 'First-export median: 3 samples; steady-state median: 21 samples (GC between exports).', + 'Peak RSS is the median process high-water mark after export, before round-trip validation.', + 'It includes setup/runtime memory and is not an isolated SQLite allocation measurement.', + '', + '| Approach | Output MiB | First export ms | Repeated export ms | Before RSS MiB | After first RSS MiB | Peak RSS MiB |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ...measurements.map(row => '| ' + [ + row.strategy, row.outputMiB.toFixed(2), row.firstMs.toFixed(2), + row.steadyMs.toFixed(2), row.beforeRssMiB.toFixed(1), + row.firstRssMiB.toFixed(1), row.peakRssMiB.toFixed(1) + ].join(' | ') + ' |'), + '', + 'Behavior and bundle sizes (including gzip) are in the JSON report above.' + ]; + const markdown = lines.join('\n') + '\n'; + console.log(markdown); + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); + } +} + +if (process.argv[2] === '--worker') { + worker(process.argv[3], process.argv[4], process.argv[5], Number(process.argv[6])) + .then(result => console.log(JSON.stringify(result))) + .catch(error => { console.error(error); process.exitCode = 1; }); +} else { + compare(); +} diff --git a/test/test_export.js b/test/test_export.js new file mode 100644 index 00000000..28049ac5 --- /dev/null +++ b/test/test_export.js @@ -0,0 +1,186 @@ +exports.test = function(SQL, assert) { + function values(db, query) { + return db.exec(query)[0].values; + } + + // A fresh database is exportable, including inside an empty transaction. + for (const transaction of [false, true]) { + const empty = new SQL.Database(); + let copy; + try { + if (transaction) empty.run('BEGIN'); + const bytes = empty.export(); + assert.ok(bytes instanceof Uint8Array); + copy = new SQL.Database(bytes); + assert.deepStrictEqual(copy.exec('SELECT * FROM sqlite_master'), []); + if (transaction) empty.run('ROLLBACK'); + empty.run('CREATE TABLE after_export(x)'); + } finally { + if (copy) copy.close(); + empty.close(); + } + } + + const db = new SQL.Database(); + let copy; + let cursor; + let iterator; + try { + db.run('PRAGMA page_size = 1024; PRAGMA foreign_keys = ON;' + + 'PRAGMA cache_size = 123; PRAGMA busy_timeout = 456;' + + 'CREATE TABLE items(id INTEGER PRIMARY KEY, data BLOB);' + + "INSERT INTO items VALUES (1, x'0080ff'), (2, NULL);" + + 'CREATE TEMP TABLE scratch(x); INSERT INTO scratch VALUES (7);' + + "ATTACH DATABASE ':memory:' AS aux;" + + 'CREATE TABLE aux.other(x); INSERT INTO aux.other VALUES (9);'); + db.create_function('twice', x => 2 * x); + db.create_aggregate('js_sum', { + init: () => 0, + step: (state, x) => state + x + }); + const calls = []; + db.updateHook((operation, database, table, rowId) => { + calls.push([operation, database, table, rowId]); + }); + cursor = db.prepare('SELECT id FROM items WHERE id >= ? ORDER BY id', [1]); + assert.strictEqual(cursor.step(), true); + assert.deepStrictEqual(cursor.get(), [1]); + iterator = db.iterateStatements('SELECT id FROM items ORDER BY id; SELECT 42'); + const iterated = iterator.next().value; + assert.strictEqual(iterated.step(), true); + assert.deepStrictEqual(iterated.get(), [1]); + const changes = db.getRowsModified(); + const rowId = values(db, 'SELECT last_insert_rowid()'); + + const bytes = db.export(); + const originalBytes = bytes.slice(); + copy = new SQL.Database(bytes); + assert.deepStrictEqual(values(copy, 'SELECT * FROM items ORDER BY id'), [ + [1, new Uint8Array([0, 128, 255])], [2, null] + ], 'The snapshot contains main-database rows and blobs'); + assert.deepStrictEqual(values(copy, 'PRAGMA page_size'), [[1024]]); + assert.throws(() => copy.exec('SELECT * FROM scratch'), /no such table/); + assert.throws(() => copy.exec('SELECT * FROM aux.other'), /no such table/); + + assert.strictEqual(db.getRowsModified(), changes); + assert.deepStrictEqual(values(db, 'SELECT last_insert_rowid()'), rowId); + assert.deepStrictEqual(values(db, 'PRAGMA foreign_keys'), [[1]]); + assert.deepStrictEqual(values(db, 'PRAGMA cache_size'), [[123]]); + assert.deepStrictEqual(values(db, 'PRAGMA busy_timeout'), [[456]]); + assert.deepStrictEqual(values(db, 'SELECT * FROM scratch'), [[7]]); + assert.deepStrictEqual(values(db, 'SELECT * FROM aux.other'), [[9]]); + assert.deepStrictEqual(values(db, 'SELECT twice(js_sum(id)) FROM items'), [[6]], + 'Custom functions and aggregates remain registered'); + assert.deepStrictEqual(values(db, "SELECT reverse('abc')"), [['cba']], + 'Extension functions remain registered'); + + assert.strictEqual(cursor.step(), true); + assert.deepStrictEqual(cursor.get(), [2], 'Cursor position and bindings survive'); + assert.strictEqual(cursor.step(), false); + cursor.free(); + cursor = null; + assert.strictEqual(iterated.step(), true); + assert.deepStrictEqual(iterated.get(), [2], 'Iterator cursor survives'); + const next = iterator.next().value; + assert.strictEqual(next.step(), true); + assert.deepStrictEqual(next.get(), [42], 'Iterator can prepare its next statement'); + assert.strictEqual(iterator.next().done, true); + iterator = null; + + assert.deepStrictEqual(calls, [], 'Export does not report updates'); + db.run('INSERT INTO items VALUES (3, NULL)'); + db.export(); + db.run('INSERT INTO items VALUES (4, NULL)'); + assert.deepStrictEqual(calls, [ + ['insert', 'main', 'items', 3], ['insert', 'main', 'items', 4] + ], 'The hook survives repeated exports'); + db.updateHook(null); + db.run('INSERT INTO items VALUES (5, NULL)'); + assert.strictEqual(calls.length, 2, 'The hook can still be unregistered'); + assert.deepStrictEqual(bytes, originalBytes, + 'Subsequent writes and exports do not change earlier snapshots'); + bytes[0] = 0; + assert.strictEqual(db.export()[0], 83, + 'Mutating an exported array does not change the database'); + } finally { + if (iterator) iterator.return(); + if (cursor) cursor.free(); + if (copy) copy.close(); + db.close(); + } + assert.throws(() => db.export(), /Database closed/); + + // Write transactions fail promptly without losing writes or savepoints. + const transactional = new SQL.Database(); + let committed; + try { + transactional.run('CREATE TABLE t(x); INSERT INTO t VALUES (1);' + + 'BEGIN; INSERT INTO t VALUES (2);' + + 'SAVEPOINT saved; INSERT INTO t VALUES (3);'); + for (let i = 0; i < 3; i++) { + assert.throws(() => transactional.export(), /busy or locked/); + } + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t ORDER BY x'), + [[1], [2], [3]], 'Failed exports leave uncommitted writes intact'); + transactional.run('ROLLBACK TO saved'); + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t ORDER BY x'), + [[1], [2]], 'Savepoints survive a failed export'); + transactional.run('ROLLBACK'); + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t'), [[1]], + 'The transaction can still be rolled back after a failed export'); + + transactional.run('BEGIN; INSERT INTO t VALUES (4)'); + assert.throws(() => transactional.export(), /busy or locked/); + transactional.run('COMMIT'); + committed = new SQL.Database(transactional.export()); + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t ORDER BY x'), + [[1], [4]], 'A failed export does not prevent commit or a later export'); + assert.deepStrictEqual(values(committed, 'SELECT x FROM t ORDER BY x'), + [[1], [4]]); + transactional.run('BEGIN IMMEDIATE'); + assert.throws(() => transactional.export(), /busy or locked/, + 'A write transaction is rejected even before it changes any rows'); + transactional.run('ROLLBACK; BEGIN'); + values(transactional, 'SELECT x FROM t'); + const readCopy = new SQL.Database(transactional.export()); + try { + assert.deepStrictEqual(values(readCopy, 'SELECT x FROM t ORDER BY x'), + [[1], [4]], 'Active read transactions can be exported'); + } finally { + readCopy.close(); + } + transactional.run('ROLLBACK'); + } finally { + if (committed) committed.close(); + transactional.close(); + } + + // Exclusive locking lets WAL work without shared-memory VFS methods. + const wal = new SQL.Database(); + let walBytes; + try { + wal.run('PRAGMA locking_mode = EXCLUSIVE'); + assert.deepStrictEqual(values(wal, 'PRAGMA journal_mode = WAL'), [['wal']]); + wal.run('PRAGMA wal_autocheckpoint = 0;' + + 'CREATE TABLE t(x); INSERT INTO t VALUES (1);' + + 'BEGIN; INSERT INTO t VALUES (2);'); + assert.throws(() => wal.export(), /busy or locked/); + wal.run('ROLLBACK'); + walBytes = wal.export(); + assert.deepStrictEqual([walBytes[18], walBytes[19]], [1, 1], + 'Exported files do not require WAL support'); + assert.deepStrictEqual(values(wal, 'PRAGMA journal_mode'), [['wal']], + 'The source keeps its journal mode'); + assert.deepStrictEqual(values(wal, 'SELECT x FROM t'), [[1]]); + } finally { + wal.close(); + } + const walCopy = new SQL.Database(walBytes); + try { + assert.deepStrictEqual(values(walCopy, 'SELECT x FROM t ORDER BY x'), + [[1]], 'A WAL snapshot opens independently after closing the source'); + walCopy.run('INSERT INTO t VALUES (3)'); + } finally { + walCopy.close(); + } +}; diff --git a/test/test_function_registration.js b/test/test_function_registration.js index ddf6eb34..d5a806ee 100644 --- a/test/test_function_registration.js +++ b/test/test_function_registration.js @@ -25,7 +25,7 @@ exports.test = function(SQL, assert) { db.create_aggregate('total', sum); db.create_function('total__finalize', () => 8); assert.deepStrictEqual(values('SELECT total(6), total__finalize()'), [[6, 8]], - 'Registrations work after export resets the registry'); + 'Registrations can be replaced after export'); } finally { db.close(); } From 7f0f5c053141ccc9baa4c1afab9c813d1c874041 Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Mon, 26 Oct 2026 09:55:30 +0000 Subject: [PATCH 2/9] test: compare large exports and constrained WASM memory in CI --- src/api.js | 4 +- test/compare_exports.cjs | 129 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 123 insertions(+), 10 deletions(-) diff --git a/src/api.js b/src/api.js index 810e22c1..cacf0ce3 100644 --- a/src/api.js +++ b/src/api.js @@ -1124,8 +1124,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { * * Exporting during an active write transaction throws an error. Commit or * roll back the transaction before exporting. Read transactions are - * allowed. Exporting never commits or rolls back a transaction or savepoint, - * including when the export fails. + * allowed. Exporting never commits or rolls back transactions or savepoints, + * even when the export fails. * * The exported file uses rollback-journal format, even if this connection * uses WAL mode. diff --git a/test/compare_exports.cjs b/test/compare_exports.cjs index d3801e7b..0ce5bd19 100644 --- a/test/compare_exports.cjs +++ b/test/compare_exports.cjs @@ -13,8 +13,74 @@ function values(db, query) { return db.exec(query)[0].values; } +function readLeb(bytes, offset) { + let value = 0; + let scale = 1; + let byte; + do { + byte = bytes[offset++]; + value += (byte & 127) * scale; + scale *= 128; + } while (byte & 128); + return { value, offset }; +} + +function leb(value) { + const bytes = []; + do { + const low = value % 128; + value = Math.floor(value / 128); + bytes.push(low | (value ? 128 : 0)); + } while (value); + return bytes; +} + +// Change only the declared memory maximum for the constrained-memory test. +// The engine enforces the cap; application code and the initial memory are +// unchanged. Normal measurements use the original, unmodified WASM bytes. +function memoryLimits(bytes, capPages) { + for (let offset = 8; offset < bytes.length;) { + const sectionStart = offset; + const id = bytes[offset++]; + const size = readLeb(bytes, offset); + const end = size.offset + size.value; + if (id === 5) { + const count = readLeb(bytes, size.offset); + const flags = readLeb(bytes, count.offset); + const initial = readLeb(bytes, flags.offset); + const maximum = readLeb(bytes, initial.offset); + assert.equal(count.value, 1); + assert.equal(flags.value, 1, 'Expected one unshared wasm32 memory with a maximum'); + assert.equal(maximum.offset, end); + if (capPages !== undefined) { + assert.ok(initial.value <= capPages && capPages <= maximum.value); + const payload = Buffer.from([1, 1, ...leb(initial.value), ...leb(capPages)]); + bytes = Buffer.concat([ + bytes.subarray(0, sectionStart), + Buffer.from([5, ...leb(payload.length)]), payload, bytes.subarray(end) + ]); + } + return { bytes, maximumMiB: (capPages ?? maximum.value) / 16 }; + } + offset = end; + } + throw new Error('The WASM module has no declared memory'); +} + async function worker(strategy, directory, mode, rows) { - const SQL = await require(path.resolve(directory, 'sql-wasm.js'))(); + const binary = memoryLimits(fs.readFileSync(path.resolve(directory, 'sql-wasm.wasm')), + mode === 'limited' ? 1024 : undefined); + let memory; + const SQL = await require(path.resolve(directory, 'sql-wasm.js'))({ + instantiateWasm(imports, receiveInstance) { + const module = new WebAssembly.Module(binary.bytes); + const instance = new WebAssembly.Instance(module, imports); + memory = Object.values(instance.exports).find(value => value instanceof WebAssembly.Memory); + assert.ok(memory, 'The module exports its linear memory'); + receiveInstance(instance, module); + return instance.exports; + } + }); const db = new SQL.Database(); try { if (mode === 'behavior') { @@ -55,7 +121,7 @@ async function worker(strategy, directory, mode, rows) { } assert.deepEqual(values(db, 'PRAGMA query_only'), [[1]]); outcomes.query_only = 'exported; setting preserved'; - return outcomes; + return { ...outcomes, maximumWasmMiB: binary.maximumMiB }; } db.run('CREATE TABLE payload(id INTEGER PRIMARY KEY, data BLOB);' @@ -63,6 +129,39 @@ async function worker(strategy, directory, mode, rows) { + 'SELECT n+1 FROM seq WHERE n < ' + rows + ') ' + 'INSERT INTO payload SELECT n, zeroblob(4096) FROM seq;' + 'CREATE INDEX payload_length ON payload(length(data));'); + const beforeWasmMiB = memory.buffer.byteLength / 1048576; + if (mode === 'limited') { + const expectedBytes = values(db, 'PRAGMA page_count')[0][0] + * values(db, 'PRAGMA page_size')[0][0]; + let bytes; + let error; + const start = performance.now(); + try { bytes = db.export(); } catch (e) { error = e; } + const elapsedMs = performance.now() - start; + if (strategy === 'serialize') { + assert.match(error && error.message, /Unable to serialize database/); + } else { + assert.ifError(error); + assert.equal(bytes.length, expectedBytes); + const copy = new SQL.Database(bytes); + try { + assert.deepEqual(values(copy, 'PRAGMA integrity_check'), [['ok']]); + assert.deepEqual(values(copy, 'SELECT count(*) FROM payload'), [[rows]]); + } finally { + copy.close(); + } + } + assert.deepEqual(values(db, 'SELECT count(*) FROM payload'), [[rows]], + 'The source remains usable after the constrained-memory export'); + return { + maximumWasmMiB: binary.maximumMiB, + databaseMiB: expectedBytes / 1048576, + beforeWasmMiB, + afterWasmMiB: memory.buffer.byteLength / 1048576, + result: error ? error.message : 'exported and integrity checked', + elapsedMs + }; + } global.gc(); const beforeRss = process.memoryUsage().rss; const beforePeak = process.resourceUsage().maxRSS * 1024; @@ -71,6 +170,7 @@ async function worker(strategy, directory, mode, rows) { const firstMs = performance.now() - start; const outputBytes = bytes.length; const firstRss = process.memoryUsage().rss; + const firstWasmMiB = memory.buffer.byteLength / 1048576; const samplesMs = []; for (let i = 0; i < 7; i++) { bytes = null; @@ -91,7 +191,8 @@ async function worker(strategy, directory, mode, rows) { } finally { copy.close(); } - return { outputBytes, firstMs, samplesMs, beforeRss, beforePeak, firstRss, peakRss }; + return { outputBytes, firstMs, samplesMs, beforeRss, beforePeak, firstRss, + peakRss, beforeWasmMiB, firstWasmMiB }; } finally { db.close(); } @@ -117,9 +218,11 @@ function compare() { backup: path.resolve(process.argv[3]) }; const behavior = {}; + const constrainedMemory = {}; const bundle = {}; for (const strategy of Object.keys(directories)) { behavior[strategy] = child(strategy, directories[strategy], 'behavior'); + constrainedMemory[strategy] = child(strategy, directories[strategy], 'limited', 24576); bundle[strategy] = {}; for (const file of ['sql-wasm.js', 'sql-wasm.wasm']) { const bytes = fs.readFileSync(path.join(directories[strategy], file)); @@ -127,7 +230,7 @@ function compare() { } } const measurements = []; - for (const rows of [256, 4096, 16384]) { + for (const rows of [256, 4096, 16384, 32768, 131072]) { const trials = { serialize: [], backup: [] }; for (let repeat = 0; repeat < 3; repeat++) { const order = repeat % 2 ? ['backup', 'serialize'] : ['serialize', 'backup']; @@ -146,13 +249,15 @@ function compare() { beforeRssMiB: median(runs.map(run => run.beforeRss / 1048576)), firstRssMiB: median(runs.map(run => run.firstRss / 1048576)), peakRssMiB: median(runs.map(run => run.peakRss / 1048576)), + beforeWasmMiB: median(runs.map(run => run.beforeWasmMiB)), + firstWasmMiB: median(runs.map(run => run.firstWasmMiB)), runs }); } } const report = { environment: { node: process.version, cpu: os.cpus()[0].model, platform: os.platform() }, - behavior, bundle, measurements + behavior, constrainedMemory, bundle, measurements }; console.log(JSON.stringify(report, null, 2)); const lines = [ @@ -163,14 +268,22 @@ function compare() { 'Peak RSS is the median process high-water mark after export, before round-trip validation.', 'It includes setup/runtime memory and is not an isolated SQLite allocation measurement.', '', - '| Approach | Output MiB | First export ms | Repeated export ms | Before RSS MiB | After first RSS MiB | Peak RSS MiB |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + '| Approach | Output MiB | First export ms | Repeated export ms | Before RSS MiB | After first RSS MiB | Peak RSS MiB | WASM before/after MiB |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', ...measurements.map(row => '| ' + [ row.strategy, row.outputMiB.toFixed(2), row.firstMs.toFixed(2), row.steadyMs.toFixed(2), row.beforeRssMiB.toFixed(1), - row.firstRssMiB.toFixed(1), row.peakRssMiB.toFixed(1) + row.firstRssMiB.toFixed(1), row.peakRssMiB.toFixed(1), + row.beforeWasmMiB.toFixed(1) + '/' + row.firstWasmMiB.toFixed(1) ].join(' | ') + ' |'), '', + '### 64 MiB hard WASM cap with a larger database', + '', + ...Object.entries(constrainedMemory).map(([strategy, result]) => + '- ' + strategy + ': ' + result.databaseMiB.toFixed(2) + ' MiB database; ' + + result.beforeWasmMiB.toFixed(1) + ' → ' + result.afterWasmMiB.toFixed(1) + + ' MiB WASM; ' + result.result + '.'), + '', 'Behavior and bundle sizes (including gzip) are in the JSON report above.' ]; const markdown = lines.join('\n') + '\n'; From e8e4065494ce7e19a79a97442a3e54b84b414d7d Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Mon, 26 Oct 2026 10:00:16 +0000 Subject: [PATCH 3/9] docs: wrap export transaction notes within the lint limit --- src/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api.js b/src/api.js index cacf0ce3..21cd7743 100644 --- a/src/api.js +++ b/src/api.js @@ -1124,8 +1124,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { * * Exporting during an active write transaction throws an error. Commit or * roll back the transaction before exporting. Read transactions are - * allowed. Exporting never commits or rolls back transactions or savepoints, - * even when the export fails. + * allowed. Exporting never commits or rolls back a transaction or changes + * savepoints, even when the export fails. * * The exported file uses rollback-journal format, even if this connection * uses WAL mode. From 5adeaa9e5c8eb3d2a42ab49801e00fc5d90c7579 Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Mon, 26 Oct 2026 10:06:34 +0000 Subject: [PATCH 4/9] ci: remove the completed cross-PR export experiment --- .github/workflows/CI.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 3b45b9e8..a272fe5e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,7 +4,6 @@ on: [push, pull_request] permissions: contents: read - actions: read jobs: build: @@ -25,13 +24,6 @@ jobs: if-no-files-found: error - name: test WebAssembly builds and distribution behavior run: npm ci && npm test - # Temporary experiment for the competing export PRs; remove after recording results. - - name: Compare backup export with the tested serialize PR - env: - GH_TOKEN: ${{ github.token }} - run: | - gh run download 40301139618 --repo sql-js/sql.js --name dist --dir "$RUNNER_TEMP/serialize-dist" - node test/compare_exports.cjs "$RUNNER_TEMP/serialize-dist" dist - name: generate documentation run: npm run doc - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 From eb6aa7cb560168bfaef0a5d79cf5dc2cced0026d Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Mon, 26 Oct 2026 10:47:47 +0000 Subject: [PATCH 5/9] perf: back up directly into a preallocated export buffer --- .github/workflows/CI.yml | 9 ++++++++ src/api.js | 34 +++++++++++++++++++++++++-- test/compare_exports.cjs | 13 +++++++---- test/test_export.js | 50 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a272fe5e..353ce705 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,6 +4,7 @@ on: [push, pull_request] permissions: contents: read + actions: read jobs: build: @@ -24,6 +25,14 @@ jobs: if-no-files-found: error - name: test WebAssembly builds and distribution behavior run: npm ci && npm test + # One-off comparison; remove after recording the optimization results. + - name: Compare optimized backup export with both baselines + env: + GH_TOKEN: ${{ github.token }} + run: | + gh run download 40301139618 --repo sql-js/sql.js --name dist --dir "$RUNNER_TEMP/serialize-dist" + gh run download 40304818305 --repo sql-js/sql.js --name dist --dir "$RUNNER_TEMP/backup-before-dist" + node test/compare_exports.cjs "$RUNNER_TEMP/serialize-dist" dist "$RUNNER_TEMP/backup-before-dist" - name: generate documentation run: npm run doc - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/src/api.js b/src/api.js index 21cd7743..3158ddf8 100644 --- a/src/api.js +++ b/src/api.js @@ -1144,6 +1144,23 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var destination = NULL; var backup = NULL; try { + var file = FS.lookupPath(filename).node; + var binaryDb = null; + var write = file.stream_ops.write; + file.stream_ops = Object.create(file.stream_ops); + file.stream_ops.write = function writeExport( + stream, buffer, offset, length, position + ) { + if (binaryDb === null) { + return write(stream, buffer, offset, length, position); + } + // Write directly into the final JS-owned result buffer. + // MEMFS would otherwise replace it on the first write. + file.contents.set(buffer.subarray(offset, offset + length), + position); + file.usedBytes = Math.max(file.usedBytes, position + length); + return length; + }; var rc = sqlite3_open(filename, apiTemp); destination = getValue(apiTemp, "i32"); if (rc !== SQLITE_OK) { @@ -1153,7 +1170,18 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { if (backup === NULL) { throw new Error(sqlite3_errmsg(destination)); } - rc = sqlite3_backup_step(backup, -1); + // Initialize the backup and reject writers before allocating. + rc = sqlite3_backup_step(backup, 0); + if (rc === SQLITE_OK) { + var sizes = this.exec( + "PRAGMA main.page_count; PRAGMA main.page_size" + ); + binaryDb = new Uint8Array( + sizes[0].values[0][0] * sizes[1].values[0][0] + ); + file.contents = binaryDb; + rc = sqlite3_backup_step(backup, -1); + } // Do not retry: this synchronous call cannot let a writer progress. if (rc === SQLITE_BUSY || rc === SQLITE_LOCKED) { throw new Error("Unable to export: SQLite is busy or locked. " @@ -1172,7 +1200,9 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { throw new Error(sqlite3_errmsg(destination)); } destination = NULL; - var binaryDb = FS.readFile(filename, { encoding: "binary" }); + // The private file is closed and unlinked below; hand its buffer + // to the caller rather than making a full-size FS.readFile copy. + binaryDb = file.contents.subarray(0, file.usedBytes); if (binaryDb.length > 0) { // All pages, including WAL pages, are in the backup. // Make the exported copy independent of WAL support. diff --git a/test/compare_exports.cjs b/test/compare_exports.cjs index 0ce5bd19..90e4ff80 100644 --- a/test/compare_exports.cjs +++ b/test/compare_exports.cjs @@ -1,4 +1,5 @@ -// CI-only comparison: node test/compare_exports.cjs SERIALIZE_DIST BACKUP_DIST +// CI-only comparison: +// node test/compare_exports.cjs SERIALIZE_DIST BACKUP_DIST [PREVIOUS_BACKUP_DIST] // Each measurement uses a fresh child process; both implementations run on // the same machine. Timing is informational, never a pass/fail threshold. const assert = require('node:assert/strict'); @@ -95,7 +96,7 @@ async function worker(strategy, directory, mode, rows) { let bytes; let error; try { bytes = db.export(); } catch (e) { error = e; } - if (strategy === 'backup' && name !== 'read_transaction') { + if (strategy !== 'serialize' && name !== 'read_transaction') { assert.match(error && error.message, /busy or locked/); outcomes[name] = 'rejected; source transaction intact'; } else { @@ -217,6 +218,10 @@ function compare() { serialize: path.resolve(process.argv[2]), backup: path.resolve(process.argv[3]) }; + if (process.argv[4]) { + directories.backup_before = path.resolve(process.argv[4]); + } + const strategies = Object.keys(directories); const behavior = {}; const constrainedMemory = {}; const bundle = {}; @@ -231,9 +236,9 @@ function compare() { } const measurements = []; for (const rows of [256, 4096, 16384, 32768, 131072]) { - const trials = { serialize: [], backup: [] }; + const trials = Object.fromEntries(strategies.map(strategy => [strategy, []])); for (let repeat = 0; repeat < 3; repeat++) { - const order = repeat % 2 ? ['backup', 'serialize'] : ['serialize', 'backup']; + const order = repeat % 2 ? strategies.slice().reverse() : strategies; for (const strategy of order) { trials[strategy].push(child(strategy, directories[strategy], 'timing', rows)); } diff --git a/test/test_export.js b/test/test_export.js index 28049ac5..f05a8785 100644 --- a/test/test_export.js +++ b/test/test_export.js @@ -110,6 +110,56 @@ exports.test = function(SQL, assert) { } assert.throws(() => db.export(), /Database closed/); + // Export fills one result buffer page by page, without recopying the file. + const large = new SQL.Database(); + let largeBytes; + try { + large.run('CREATE TABLE t(data); INSERT INTO t VALUES (zeroblob(3145728))'); + const set = Uint8Array.prototype.set; + let largestCopy = 0; + Uint8Array.prototype.set = function(source, offset) { + largestCopy = Math.max(largestCopy, source.length); + return set.call(this, source, offset); + }; + try { + largeBytes = large.export(); + } finally { + Uint8Array.prototype.set = set; + } + assert.ok(largestCopy <= 65536, + 'Export never copies a growing file or a full-size result array'); + assert.strictEqual(largeBytes.buffer.byteLength, largeBytes.byteLength, + 'The result owns an exactly sized buffer, not spare MEMFS capacity'); + } finally { + large.close(); + } + const largeCopy = new SQL.Database(largeBytes); + try { + assert.deepStrictEqual(values(largeCopy, 'PRAGMA integrity_check'), [['ok']]); + assert.deepStrictEqual(values(largeCopy, 'SELECT length(data) FROM t'), + [[3145728]], 'The result remains valid after closing the source'); + } finally { + largeCopy.close(); + } + + for (const pageSize of [512, 4096, 65536]) { + const paged = new SQL.Database(); + let pagedCopy; + try { + paged.run('PRAGMA page_size = ' + pageSize + ';' + + 'CREATE TABLE t(x); INSERT INTO t VALUES (42);'); + const bytes = paged.export(); + assert.strictEqual(bytes.length, + values(paged, 'PRAGMA page_count')[0][0] * pageSize); + assert.strictEqual(bytes.buffer.byteLength, bytes.length); + pagedCopy = new SQL.Database(bytes); + assert.deepStrictEqual(values(pagedCopy, 'SELECT x FROM t'), [[42]]); + } finally { + if (pagedCopy) pagedCopy.close(); + paged.close(); + } + } + // Write transactions fail promptly without losing writes or savepoints. const transactional = new SQL.Database(); let committed; From 37f37c8aef99df56d29162879436a9abbf923d30 Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Mon, 26 Oct 2026 10:51:28 +0000 Subject: [PATCH 6/9] fix: return unsigned views for empty backup snapshots --- src/api.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/api.js b/src/api.js index 3158ddf8..c01f9523 100644 --- a/src/api.js +++ b/src/api.js @@ -1202,7 +1202,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { destination = NULL; // The private file is closed and unlinked below; hand its buffer // to the caller rather than making a full-size FS.readFile copy. - binaryDb = file.contents.subarray(0, file.usedBytes); + binaryDb = new Uint8Array(file.contents.buffer, + file.contents.byteOffset, file.usedBytes); if (binaryDb.length > 0) { // All pages, including WAL pages, are in the backup. // Make the exported copy independent of WAL support. From bf9de3dbe2e24d175cc406541c717ee7a6674251 Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Mon, 26 Oct 2026 10:55:38 +0000 Subject: [PATCH 7/9] test: restore typed-array instrumentation without prototype pollution --- test/test_export.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/test_export.js b/test/test_export.js index f05a8785..da67b90a 100644 --- a/test/test_export.js +++ b/test/test_export.js @@ -115,16 +115,17 @@ exports.test = function(SQL, assert) { let largeBytes; try { large.run('CREATE TABLE t(data); INSERT INTO t VALUES (zeroblob(3145728))'); - const set = Uint8Array.prototype.set; + const prototype = Object.getPrototypeOf(Uint8Array.prototype); + const set = prototype.set; let largestCopy = 0; - Uint8Array.prototype.set = function(source, offset) { + prototype.set = function(source, offset) { largestCopy = Math.max(largestCopy, source.length); return set.call(this, source, offset); }; try { largeBytes = large.export(); } finally { - Uint8Array.prototype.set = set; + prototype.set = set; } assert.ok(largestCopy <= 65536, 'Export never copies a growing file or a full-size result array'); From 1b2830c0be89072ed93522edf495c2ee5ce6de06 Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Mon, 26 Oct 2026 11:02:30 +0000 Subject: [PATCH 8/9] ci: remove the completed export optimization comparison --- .github/workflows/CI.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 353ce705..a272fe5e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,7 +4,6 @@ on: [push, pull_request] permissions: contents: read - actions: read jobs: build: @@ -25,14 +24,6 @@ jobs: if-no-files-found: error - name: test WebAssembly builds and distribution behavior run: npm ci && npm test - # One-off comparison; remove after recording the optimization results. - - name: Compare optimized backup export with both baselines - env: - GH_TOKEN: ${{ github.token }} - run: | - gh run download 40301139618 --repo sql-js/sql.js --name dist --dir "$RUNNER_TEMP/serialize-dist" - gh run download 40304818305 --repo sql-js/sql.js --name dist --dir "$RUNNER_TEMP/backup-before-dist" - node test/compare_exports.cjs "$RUNNER_TEMP/serialize-dist" dist "$RUNNER_TEMP/backup-before-dist" - name: generate documentation run: npm run doc - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 From a7e5d6d8bb496d8c4f9b3658bb629d24395468b0 Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Mon, 26 Oct 2026 11:06:25 +0000 Subject: [PATCH 9/9] chore: clean up export experiment and prepare 1.15.2 --- package-lock.json | 4 +- package.json | 2 +- test/compare_exports.cjs | 307 --------------------------------------- 3 files changed, 3 insertions(+), 310 deletions(-) delete mode 100644 test/compare_exports.cjs diff --git a/package-lock.json b/package-lock.json index 07f24ff2..a2342b38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "sql.js", - "version": "1.15.1", + "version": "1.15.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sql.js", - "version": "1.15.1", + "version": "1.15.2", "license": "MIT", "devDependencies": { "@eslint/js": "^10.0.1", diff --git a/package.json b/package.json index 27622a79..4c1e9d46 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sql.js", - "version": "1.15.1", + "version": "1.15.2", "description": "SQLite library with support for opening and writing databases, prepared statements, and more. Compiled to WebAssembly with Emscripten, with JavaScript bindings.", "keywords": [ "sql", diff --git a/test/compare_exports.cjs b/test/compare_exports.cjs deleted file mode 100644 index 90e4ff80..00000000 --- a/test/compare_exports.cjs +++ /dev/null @@ -1,307 +0,0 @@ -// CI-only comparison: -// node test/compare_exports.cjs SERIALIZE_DIST BACKUP_DIST [PREVIOUS_BACKUP_DIST] -// Each measurement uses a fresh child process; both implementations run on -// the same machine. Timing is informational, never a pass/fail threshold. -const assert = require('node:assert/strict'); -const fs = require('node:fs'); -const os = require('node:os'); -const path = require('node:path'); -const { spawnSync } = require('node:child_process'); -const { performance } = require('node:perf_hooks'); -const { gzipSync } = require('node:zlib'); - -function values(db, query) { - return db.exec(query)[0].values; -} - -function readLeb(bytes, offset) { - let value = 0; - let scale = 1; - let byte; - do { - byte = bytes[offset++]; - value += (byte & 127) * scale; - scale *= 128; - } while (byte & 128); - return { value, offset }; -} - -function leb(value) { - const bytes = []; - do { - const low = value % 128; - value = Math.floor(value / 128); - bytes.push(low | (value ? 128 : 0)); - } while (value); - return bytes; -} - -// Change only the declared memory maximum for the constrained-memory test. -// The engine enforces the cap; application code and the initial memory are -// unchanged. Normal measurements use the original, unmodified WASM bytes. -function memoryLimits(bytes, capPages) { - for (let offset = 8; offset < bytes.length;) { - const sectionStart = offset; - const id = bytes[offset++]; - const size = readLeb(bytes, offset); - const end = size.offset + size.value; - if (id === 5) { - const count = readLeb(bytes, size.offset); - const flags = readLeb(bytes, count.offset); - const initial = readLeb(bytes, flags.offset); - const maximum = readLeb(bytes, initial.offset); - assert.equal(count.value, 1); - assert.equal(flags.value, 1, 'Expected one unshared wasm32 memory with a maximum'); - assert.equal(maximum.offset, end); - if (capPages !== undefined) { - assert.ok(initial.value <= capPages && capPages <= maximum.value); - const payload = Buffer.from([1, 1, ...leb(initial.value), ...leb(capPages)]); - bytes = Buffer.concat([ - bytes.subarray(0, sectionStart), - Buffer.from([5, ...leb(payload.length)]), payload, bytes.subarray(end) - ]); - } - return { bytes, maximumMiB: (capPages ?? maximum.value) / 16 }; - } - offset = end; - } - throw new Error('The WASM module has no declared memory'); -} - -async function worker(strategy, directory, mode, rows) { - const binary = memoryLimits(fs.readFileSync(path.resolve(directory, 'sql-wasm.wasm')), - mode === 'limited' ? 1024 : undefined); - let memory; - const SQL = await require(path.resolve(directory, 'sql-wasm.js'))({ - instantiateWasm(imports, receiveInstance) { - const module = new WebAssembly.Module(binary.bytes); - const instance = new WebAssembly.Instance(module, imports); - memory = Object.values(instance.exports).find(value => value instanceof WebAssembly.Memory); - assert.ok(memory, 'The module exports its linear memory'); - receiveInstance(instance, module); - return instance.exports; - } - }); - const db = new SQL.Database(); - try { - if (mode === 'behavior') { - const outcomes = {}; - db.run('CREATE TABLE t(x); INSERT INTO t VALUES (1)'); - for (const [name, begin, expected] of [ - ['read_transaction', 'BEGIN; SELECT * FROM t', [[1]]], - ['reserved_write_transaction', 'BEGIN IMMEDIATE', [[1]]], - ['uncommitted_writes', 'BEGIN; INSERT INTO t VALUES (2)', [[1], [2]]] - ]) { - db.run(begin); - let bytes; - let error; - try { bytes = db.export(); } catch (e) { error = e; } - if (strategy !== 'serialize' && name !== 'read_transaction') { - assert.match(error && error.message, /busy or locked/); - outcomes[name] = 'rejected; source transaction intact'; - } else { - assert.ifError(error); - const copy = new SQL.Database(bytes); - try { - assert.deepEqual(values(copy, 'SELECT x FROM t ORDER BY x'), expected); - } finally { - copy.close(); - } - outcomes[name] = 'exported; source transaction intact'; - } - assert.deepEqual(values(db, 'SELECT x FROM t ORDER BY x'), expected); - db.run('ROLLBACK'); - assert.deepEqual(values(db, 'SELECT x FROM t'), [[1]]); - } - db.run('PRAGMA query_only = ON'); - const copy = new SQL.Database(db.export()); - try { - assert.deepEqual(values(copy, 'SELECT x FROM t'), [[1]]); - } finally { - copy.close(); - } - assert.deepEqual(values(db, 'PRAGMA query_only'), [[1]]); - outcomes.query_only = 'exported; setting preserved'; - return { ...outcomes, maximumWasmMiB: binary.maximumMiB }; - } - - db.run('CREATE TABLE payload(id INTEGER PRIMARY KEY, data BLOB);' - + 'WITH RECURSIVE seq(n) AS (VALUES(1) UNION ALL ' - + 'SELECT n+1 FROM seq WHERE n < ' + rows + ') ' - + 'INSERT INTO payload SELECT n, zeroblob(4096) FROM seq;' - + 'CREATE INDEX payload_length ON payload(length(data));'); - const beforeWasmMiB = memory.buffer.byteLength / 1048576; - if (mode === 'limited') { - const expectedBytes = values(db, 'PRAGMA page_count')[0][0] - * values(db, 'PRAGMA page_size')[0][0]; - let bytes; - let error; - const start = performance.now(); - try { bytes = db.export(); } catch (e) { error = e; } - const elapsedMs = performance.now() - start; - if (strategy === 'serialize') { - assert.match(error && error.message, /Unable to serialize database/); - } else { - assert.ifError(error); - assert.equal(bytes.length, expectedBytes); - const copy = new SQL.Database(bytes); - try { - assert.deepEqual(values(copy, 'PRAGMA integrity_check'), [['ok']]); - assert.deepEqual(values(copy, 'SELECT count(*) FROM payload'), [[rows]]); - } finally { - copy.close(); - } - } - assert.deepEqual(values(db, 'SELECT count(*) FROM payload'), [[rows]], - 'The source remains usable after the constrained-memory export'); - return { - maximumWasmMiB: binary.maximumMiB, - databaseMiB: expectedBytes / 1048576, - beforeWasmMiB, - afterWasmMiB: memory.buffer.byteLength / 1048576, - result: error ? error.message : 'exported and integrity checked', - elapsedMs - }; - } - global.gc(); - const beforeRss = process.memoryUsage().rss; - const beforePeak = process.resourceUsage().maxRSS * 1024; - let start = performance.now(); - let bytes = db.export(); - const firstMs = performance.now() - start; - const outputBytes = bytes.length; - const firstRss = process.memoryUsage().rss; - const firstWasmMiB = memory.buffer.byteLength / 1048576; - const samplesMs = []; - for (let i = 0; i < 7; i++) { - bytes = null; - global.gc(); - start = performance.now(); - bytes = db.export(); - samplesMs.push(performance.now() - start); - assert.equal(bytes.length, outputBytes); - } - // Capture export-only memory before opening the independent copy. - const peakRss = process.resourceUsage().maxRSS * 1024; - const copy = new SQL.Database(bytes); - try { - assert.deepEqual(values(copy, 'PRAGMA integrity_check'), [['ok']]); - assert.deepEqual(values(copy, - 'SELECT count(*), sum(length(data)), min(id), max(id) FROM payload'), - [[rows, rows * 4096, 1, rows]]); - } finally { - copy.close(); - } - return { outputBytes, firstMs, samplesMs, beforeRss, beforePeak, firstRss, - peakRss, beforeWasmMiB, firstWasmMiB }; - } finally { - db.close(); - } -} - -function median(numbers) { - const sorted = numbers.slice().sort((a, b) => a - b); - return sorted[Math.floor(sorted.length / 2)]; -} - -function child(strategy, directory, mode, rows = 0) { - const result = spawnSync(process.execPath, [ - '--expose-gc', __filename, '--worker', strategy, directory, mode, String(rows) - ], { encoding: 'utf8', timeout: 120000, maxBuffer: 1024 * 1024 }); - if (result.error) throw result.error; - assert.equal(result.status, 0, result.stderr || result.stdout); - return JSON.parse(result.stdout); -} - -function compare() { - const directories = { - serialize: path.resolve(process.argv[2]), - backup: path.resolve(process.argv[3]) - }; - if (process.argv[4]) { - directories.backup_before = path.resolve(process.argv[4]); - } - const strategies = Object.keys(directories); - const behavior = {}; - const constrainedMemory = {}; - const bundle = {}; - for (const strategy of Object.keys(directories)) { - behavior[strategy] = child(strategy, directories[strategy], 'behavior'); - constrainedMemory[strategy] = child(strategy, directories[strategy], 'limited', 24576); - bundle[strategy] = {}; - for (const file of ['sql-wasm.js', 'sql-wasm.wasm']) { - const bytes = fs.readFileSync(path.join(directories[strategy], file)); - bundle[strategy][file] = { bytes: bytes.length, gzipBytes: gzipSync(bytes).length }; - } - } - const measurements = []; - for (const rows of [256, 4096, 16384, 32768, 131072]) { - const trials = Object.fromEntries(strategies.map(strategy => [strategy, []])); - for (let repeat = 0; repeat < 3; repeat++) { - const order = repeat % 2 ? strategies.slice().reverse() : strategies; - for (const strategy of order) { - trials[strategy].push(child(strategy, directories[strategy], 'timing', rows)); - } - } - for (const strategy of Object.keys(trials)) { - const runs = trials[strategy]; - measurements.push({ - strategy, - rows, - outputMiB: runs[0].outputBytes / 1048576, - firstMs: median(runs.map(run => run.firstMs)), - steadyMs: median(runs.flatMap(run => run.samplesMs)), - beforeRssMiB: median(runs.map(run => run.beforeRss / 1048576)), - firstRssMiB: median(runs.map(run => run.firstRss / 1048576)), - peakRssMiB: median(runs.map(run => run.peakRss / 1048576)), - beforeWasmMiB: median(runs.map(run => run.beforeWasmMiB)), - firstWasmMiB: median(runs.map(run => run.firstWasmMiB)), - runs - }); - } - } - const report = { - environment: { node: process.version, cpu: os.cpus()[0].model, platform: os.platform() }, - behavior, constrainedMemory, bundle, measurements - }; - console.log(JSON.stringify(report, null, 2)); - const lines = [ - '## Export implementation comparison', - '', - 'Same runner, optimized wasm, 3 fresh processes per strategy/size.', - 'First-export median: 3 samples; steady-state median: 21 samples (GC between exports).', - 'Peak RSS is the median process high-water mark after export, before round-trip validation.', - 'It includes setup/runtime memory and is not an isolated SQLite allocation measurement.', - '', - '| Approach | Output MiB | First export ms | Repeated export ms | Before RSS MiB | After first RSS MiB | Peak RSS MiB | WASM before/after MiB |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', - ...measurements.map(row => '| ' + [ - row.strategy, row.outputMiB.toFixed(2), row.firstMs.toFixed(2), - row.steadyMs.toFixed(2), row.beforeRssMiB.toFixed(1), - row.firstRssMiB.toFixed(1), row.peakRssMiB.toFixed(1), - row.beforeWasmMiB.toFixed(1) + '/' + row.firstWasmMiB.toFixed(1) - ].join(' | ') + ' |'), - '', - '### 64 MiB hard WASM cap with a larger database', - '', - ...Object.entries(constrainedMemory).map(([strategy, result]) => - '- ' + strategy + ': ' + result.databaseMiB.toFixed(2) + ' MiB database; ' - + result.beforeWasmMiB.toFixed(1) + ' → ' + result.afterWasmMiB.toFixed(1) - + ' MiB WASM; ' + result.result + '.'), - '', - 'Behavior and bundle sizes (including gzip) are in the JSON report above.' - ]; - const markdown = lines.join('\n') + '\n'; - console.log(markdown); - if (process.env.GITHUB_STEP_SUMMARY) { - fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); - } -} - -if (process.argv[2] === '--worker') { - worker(process.argv[3], process.argv[4], process.argv[5], Number(process.argv[6])) - .then(result => console.log(JSON.stringify(result))) - .catch(error => { console.error(error); process.exitCode = 1; }); -} else { - compare(); -}