From 0aa9423c8cfa442afab5006748efa59ecc4fa958 Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Mon, 26 Oct 2026 09:38:04 +0000 Subject: [PATCH] fix: export snapshots without reopening the database --- src/api.js | 65 ++++++++--- src/exported_functions.json | 1 + test/test_export.js | 175 +++++++++++++++++++++++++++++ test/test_function_registration.js | 2 +- 4 files changed, 228 insertions(+), 15 deletions(-) create mode 100644 test/test_export.js diff --git a/src/api.js b/src/api.js index 864607e5..5ba5873f 100644 --- a/src/api.js +++ b/src/api.js @@ -1,6 +1,7 @@ /* global FS HEAP8 + HEAPU8 Module _malloc _free @@ -75,6 +76,12 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // var - cwrap function var sqlite3_open = cwrap("sqlite3_open", "number", ["string", "number"]); var sqlite3_close_v2 = cwrap("sqlite3_close_v2", "number", ["number"]); + var sqlite3_serialize = cwrap( + "sqlite3_serialize", + "number", + ["number", "string", "number", "number"] + ); + var sqlite3_free = cwrap("sqlite3_free", "", ["number"]); var sqlite3_exec = cwrap( "sqlite3_exec", "number", @@ -1103,23 +1110,53 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { return new StatementIterator(sql, this); }; - /** Exports the contents of the database to a binary array. This - * operation will close and re-open the database which will cause - * any pragmas to be set back to their default values. + /** Exports a snapshot of the main database to a binary array without + * closing the connection. Prepared statements, registered functions, + * update hooks and connection settings (including pragmas) are preserved. + * TEMP objects and attached databases remain available on this connection, + * but are not included in the exported file. + * + * The snapshot includes uncommitted changes visible to this connection. + * Exporting does not commit or roll back an open transaction or savepoint; + * it can still be committed or rolled back afterwards. Rolling back the + * original database does not change an already exported snapshot. + * + * The exported file uses rollback-journal format, even if this connection + * uses WAL mode. + * + * The returned array is an independent copy. It can be used to create a + * new Database, but JavaScript functions, hooks and connection settings + * are not stored in the file and must be configured on the new connection. @return {Uint8Array} An array of bytes of the SQLite3 database file */ Database.prototype.export = function exportDatabase() { - Object.values(this.statements).forEach(function each(stmt) { - stmt.free(); - }); - this.functions.forEach(removeFunctionPointers); - this.functions.clear(); - this.handleError(sqlite3_close_v2(this.db)); - var binaryDb = FS.readFile(this.filename, { encoding: "binary" }); - this.handleError(sqlite3_open(this.filename, apiTemp)); - this.db = getValue(apiTemp, "i32"); - registerExtensionFunctions(this.db); - return binaryDb; + if (!this.db) { + throw new Error("Database closed"); + } + var stack = stackSave(); + var buffer = NULL; + try { + var sizePtr = stackAlloc(8); + buffer = sqlite3_serialize(this.db, "main", sizePtr, 0) >>> 0; + // Read sqlite3_int64 without requiring BigInt support. + var size = (getValue(sizePtr, "i32") >>> 0) + + getValue(sizePtr + 4, "i32") * 4294967296; + // An empty database can have a zero size and a NULL buffer. + if (buffer === NULL && size !== 0) { + throw new Error("Unable to serialize database"); + } + var binaryDb = HEAPU8.slice(buffer, buffer + size); + if (binaryDb.length > 0) { + // All pages, including WAL pages, are in this snapshot. + // Use rollback-journal format so no WAL file is required. + binaryDb[18] = 1; + binaryDb[19] = 1; + } + return binaryDb; + } finally { + sqlite3_free(buffer); + stackRestore(stack); + } }; /** Close the database, and all associated prepared statements. diff --git a/src/exported_functions.json b/src/exported_functions.json index 3be25955..7a4d2390 100644 --- a/src/exported_functions.json +++ b/src/exported_functions.json @@ -2,6 +2,7 @@ "_malloc", "_free", "_sqlite3_open", +"_sqlite3_serialize", "_sqlite3_exec", "_sqlite3_free", "_sqlite3_errmsg", diff --git a/test/test_export.js b/test/test_export.js new file mode 100644 index 00000000..03d522a2 --- /dev/null +++ b/test/test_export.js @@ -0,0 +1,175 @@ +exports.test = function(SQL, assert) { + function values(db, query) { + return db.exec(query)[0].values; + } + + // A fresh database is exportable, including inside an empty transaction. + for (const transaction of [false, true]) { + const empty = new SQL.Database(); + let copy; + try { + if (transaction) empty.run('BEGIN'); + const bytes = empty.export(); + assert.ok(bytes instanceof Uint8Array); + copy = new SQL.Database(bytes); + assert.deepStrictEqual(copy.exec('SELECT * FROM sqlite_master'), []); + if (transaction) empty.run('ROLLBACK'); + empty.run('CREATE TABLE after_export(x)'); + } finally { + if (copy) copy.close(); + empty.close(); + } + } + + const db = new SQL.Database(); + let copy; + let cursor; + let iterator; + try { + db.run('PRAGMA page_size = 1024; PRAGMA foreign_keys = ON;' + + 'PRAGMA cache_size = 123; PRAGMA busy_timeout = 456;' + + 'CREATE TABLE items(id INTEGER PRIMARY KEY, data BLOB);' + + "INSERT INTO items VALUES (1, x'0080ff'), (2, NULL);" + + 'CREATE TEMP TABLE scratch(x); INSERT INTO scratch VALUES (7);' + + "ATTACH DATABASE ':memory:' AS aux;" + + 'CREATE TABLE aux.other(x); INSERT INTO aux.other VALUES (9);'); + db.create_function('twice', x => 2 * x); + db.create_aggregate('js_sum', { + init: () => 0, + step: (state, x) => state + x + }); + const calls = []; + db.updateHook((operation, database, table, rowId) => { + calls.push([operation, database, table, rowId]); + }); + cursor = db.prepare('SELECT id FROM items WHERE id >= ? ORDER BY id', [1]); + assert.strictEqual(cursor.step(), true); + assert.deepStrictEqual(cursor.get(), [1]); + iterator = db.iterateStatements('SELECT id FROM items ORDER BY id; SELECT 42'); + const iterated = iterator.next().value; + assert.strictEqual(iterated.step(), true); + assert.deepStrictEqual(iterated.get(), [1]); + const changes = db.getRowsModified(); + const rowId = values(db, 'SELECT last_insert_rowid()'); + + const bytes = db.export(); + const originalBytes = bytes.slice(); + copy = new SQL.Database(bytes); + assert.deepStrictEqual(values(copy, 'SELECT * FROM items ORDER BY id'), [ + [1, new Uint8Array([0, 128, 255])], [2, null] + ], 'The snapshot contains main-database rows and blobs'); + assert.deepStrictEqual(values(copy, 'PRAGMA page_size'), [[1024]]); + assert.throws(() => copy.exec('SELECT * FROM scratch'), /no such table/); + assert.throws(() => copy.exec('SELECT * FROM aux.other'), /no such table/); + + assert.strictEqual(db.getRowsModified(), changes); + assert.deepStrictEqual(values(db, 'SELECT last_insert_rowid()'), rowId); + assert.deepStrictEqual(values(db, 'PRAGMA foreign_keys'), [[1]]); + assert.deepStrictEqual(values(db, 'PRAGMA cache_size'), [[123]]); + assert.deepStrictEqual(values(db, 'PRAGMA busy_timeout'), [[456]]); + assert.deepStrictEqual(values(db, 'SELECT * FROM scratch'), [[7]]); + assert.deepStrictEqual(values(db, 'SELECT * FROM aux.other'), [[9]]); + assert.deepStrictEqual(values(db, 'SELECT twice(js_sum(id)) FROM items'), [[6]], + 'Custom functions and aggregates remain registered'); + assert.deepStrictEqual(values(db, "SELECT reverse('abc')"), [['cba']], + 'Extension functions remain registered'); + + assert.strictEqual(cursor.step(), true); + assert.deepStrictEqual(cursor.get(), [2], 'Cursor position and bindings survive'); + assert.strictEqual(cursor.step(), false); + cursor.free(); + cursor = null; + assert.strictEqual(iterated.step(), true); + assert.deepStrictEqual(iterated.get(), [2], 'Iterator cursor survives'); + const next = iterator.next().value; + assert.strictEqual(next.step(), true); + assert.deepStrictEqual(next.get(), [42], 'Iterator can prepare its next statement'); + assert.strictEqual(iterator.next().done, true); + iterator = null; + + assert.deepStrictEqual(calls, [], 'Export does not report updates'); + db.run('INSERT INTO items VALUES (3, NULL)'); + db.export(); + db.run('INSERT INTO items VALUES (4, NULL)'); + assert.deepStrictEqual(calls, [ + ['insert', 'main', 'items', 3], ['insert', 'main', 'items', 4] + ], 'The hook survives repeated exports'); + db.updateHook(null); + db.run('INSERT INTO items VALUES (5, NULL)'); + assert.strictEqual(calls.length, 2, 'The hook can still be unregistered'); + assert.deepStrictEqual(bytes, originalBytes, + 'Subsequent writes and exports do not change earlier snapshots'); + bytes[0] = 0; + assert.strictEqual(db.export()[0], 83, + 'Mutating an exported array does not change the database'); + } finally { + if (iterator) iterator.return(); + if (cursor) cursor.free(); + if (copy) copy.close(); + db.close(); + } + assert.throws(() => db.export(), /Database closed/); + + // A snapshot includes uncommitted changes without ending the transaction. + const transactional = new SQL.Database(); + let snapshot; + let committed; + try { + transactional.run('CREATE TABLE t(x); INSERT INTO t VALUES (1);' + + 'BEGIN; INSERT INTO t VALUES (2);' + + 'SAVEPOINT saved; INSERT INTO t VALUES (3);'); + const bytes = transactional.export(); + snapshot = new SQL.Database(bytes); + assert.deepStrictEqual(values(snapshot, 'SELECT x FROM t ORDER BY x'), + [[1], [2], [3]], 'Export includes uncommitted changes'); + transactional.run('ROLLBACK TO saved'); + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t ORDER BY x'), + [[1], [2]], 'Savepoints survive export'); + transactional.run('ROLLBACK'); + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t'), [[1]], + 'The transaction can still be rolled back after export'); + assert.deepStrictEqual(values(snapshot, 'SELECT x FROM t ORDER BY x'), + [[1], [2], [3]], 'Rollback does not change the exported snapshot'); + + transactional.run('BEGIN; INSERT INTO t VALUES (4)'); + const committedBytes = transactional.export(); + transactional.run('COMMIT'); + committed = new SQL.Database(committedBytes); + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t ORDER BY x'), + [[1], [4]], 'The transaction can still be committed after export'); + assert.deepStrictEqual(values(committed, 'SELECT x FROM t ORDER BY x'), + [[1], [4]]); + } finally { + if (committed) committed.close(); + if (snapshot) snapshot.close(); + transactional.close(); + } + + // Exclusive locking lets WAL work without shared-memory VFS methods. + const wal = new SQL.Database(); + let walBytes; + try { + wal.run('PRAGMA locking_mode = EXCLUSIVE'); + assert.deepStrictEqual(values(wal, 'PRAGMA journal_mode = WAL'), [['wal']]); + wal.run('PRAGMA wal_autocheckpoint = 0;' + + 'CREATE TABLE t(x); INSERT INTO t VALUES (1);' + + 'BEGIN; INSERT INTO t VALUES (2);'); + walBytes = wal.export(); + assert.deepStrictEqual([walBytes[18], walBytes[19]], [1, 1], + 'Exported files do not require WAL support'); + assert.deepStrictEqual(values(wal, 'PRAGMA journal_mode'), [['wal']], + 'The source keeps its journal mode'); + wal.run('ROLLBACK'); + assert.deepStrictEqual(values(wal, 'SELECT x FROM t'), [[1]]); + } finally { + wal.close(); + } + const walCopy = new SQL.Database(walBytes); + try { + assert.deepStrictEqual(values(walCopy, 'SELECT x FROM t ORDER BY x'), + [[1], [2]], 'A WAL snapshot opens independently after closing the source'); + walCopy.run('INSERT INTO t VALUES (3)'); + } finally { + walCopy.close(); + } +}; diff --git a/test/test_function_registration.js b/test/test_function_registration.js index ddf6eb34..d5a806ee 100644 --- a/test/test_function_registration.js +++ b/test/test_function_registration.js @@ -25,7 +25,7 @@ exports.test = function(SQL, assert) { db.create_aggregate('total', sum); db.create_function('total__finalize', () => 8); assert.deepStrictEqual(values('SELECT total(6), total__finalize()'), [[6, 8]], - 'Registrations work after export resets the registry'); + 'Registrations can be replaced after export'); } finally { db.close(); }