diff --git a/package-lock.json b/package-lock.json index 07f24ff2..a2342b38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "sql.js", - "version": "1.15.1", + "version": "1.15.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sql.js", - "version": "1.15.1", + "version": "1.15.2", "license": "MIT", "devDependencies": { "@eslint/js": "^10.0.1", diff --git a/package.json b/package.json index 27622a79..4c1e9d46 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sql.js", - "version": "1.15.1", + "version": "1.15.2", "description": "SQLite library with support for opening and writing databases, prepared statements, and more. Compiled to WebAssembly with Emscripten, with JavaScript bindings.", "keywords": [ "sql", diff --git a/src/api.js b/src/api.js index 864607e5..c01f9523 100644 --- a/src/api.js +++ b/src/api.js @@ -60,6 +60,8 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var NULL = 0; // SQLite enum var SQLITE_OK = 0; + var SQLITE_BUSY = 5; + var SQLITE_LOCKED = 6; var SQLITE_ROW = 100; var SQLITE_DONE = 101; var SQLITE_INTEGER = 1; @@ -75,6 +77,17 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // var - cwrap function var sqlite3_open = cwrap("sqlite3_open", "number", ["string", "number"]); var sqlite3_close_v2 = cwrap("sqlite3_close_v2", "number", ["number"]); + var sqlite3_backup_init = cwrap( + "sqlite3_backup_init", + "number", + ["number", "string", "number", "string"] + ); + var sqlite3_backup_step = cwrap( + "sqlite3_backup_step", "number", ["number", "number"] + ); + var sqlite3_backup_finish = cwrap( + "sqlite3_backup_finish", "number", ["number"] + ); var sqlite3_exec = cwrap( "sqlite3_exec", "number", @@ -1103,23 +1116,106 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { return new StatementIterator(sql, this); }; - /** Exports the contents of the database to a binary array. This - * operation will close and re-open the database which will cause - * any pragmas to be set back to their default values. + /** Exports a snapshot of the main database to a binary array without + * closing the connection. Prepared statements, registered functions, + * update hooks and connection settings (including pragmas) are preserved. + * TEMP objects and attached databases remain available on this connection, + * but are not included in the exported file. + * + * Exporting during an active write transaction throws an error. Commit or + * roll back the transaction before exporting. Read transactions are + * allowed. Exporting never commits or rolls back a transaction or changes + * savepoints, even when the export fails. + * + * The exported file uses rollback-journal format, even if this connection + * uses WAL mode. + * + * The returned array is an independent copy. It can be used to create a + * new Database, but JavaScript functions, hooks and connection settings + * are not stored in the file and must be configured on the new connection. @return {Uint8Array} An array of bytes of the SQLite3 database file */ Database.prototype.export = function exportDatabase() { - Object.values(this.statements).forEach(function each(stmt) { - stmt.free(); - }); - this.functions.forEach(removeFunctionPointers); - this.functions.clear(); - this.handleError(sqlite3_close_v2(this.db)); - var binaryDb = FS.readFile(this.filename, { encoding: "binary" }); - this.handleError(sqlite3_open(this.filename, apiTemp)); - this.db = getValue(apiTemp, "i32"); - registerExtensionFunctions(this.db); - return binaryDb; + if (!this.db) { + throw new Error("Database closed"); + } + var filename = this.filename + "_export"; + FS.createDataFile("/", filename, [], true, true); + var destination = NULL; + var backup = NULL; + try { + var file = FS.lookupPath(filename).node; + var binaryDb = null; + var write = file.stream_ops.write; + file.stream_ops = Object.create(file.stream_ops); + file.stream_ops.write = function writeExport( + stream, buffer, offset, length, position + ) { + if (binaryDb === null) { + return write(stream, buffer, offset, length, position); + } + // Write directly into the final JS-owned result buffer. + // MEMFS would otherwise replace it on the first write. + file.contents.set(buffer.subarray(offset, offset + length), + position); + file.usedBytes = Math.max(file.usedBytes, position + length); + return length; + }; + var rc = sqlite3_open(filename, apiTemp); + destination = getValue(apiTemp, "i32"); + if (rc !== SQLITE_OK) { + throw new Error(sqlite3_errmsg(destination)); + } + backup = sqlite3_backup_init(destination, "main", this.db, "main"); + if (backup === NULL) { + throw new Error(sqlite3_errmsg(destination)); + } + // Initialize the backup and reject writers before allocating. + rc = sqlite3_backup_step(backup, 0); + if (rc === SQLITE_OK) { + var sizes = this.exec( + "PRAGMA main.page_count; PRAGMA main.page_size" + ); + binaryDb = new Uint8Array( + sizes[0].values[0][0] * sizes[1].values[0][0] + ); + file.contents = binaryDb; + rc = sqlite3_backup_step(backup, -1); + } + // Do not retry: this synchronous call cannot let a writer progress. + if (rc === SQLITE_BUSY || rc === SQLITE_LOCKED) { + throw new Error("Unable to export: SQLite is busy or locked. " + + "Finish active write transactions before exporting."); + } + if (rc !== SQLITE_DONE) { + throw new Error(sqlite3_errmsg(destination)); + } + rc = sqlite3_backup_finish(backup); + backup = NULL; + if (rc !== SQLITE_OK) { + throw new Error(sqlite3_errmsg(destination)); + } + rc = sqlite3_close_v2(destination); + if (rc !== SQLITE_OK) { + throw new Error(sqlite3_errmsg(destination)); + } + destination = NULL; + // The private file is closed and unlinked below; hand its buffer + // to the caller rather than making a full-size FS.readFile copy. + binaryDb = new Uint8Array(file.contents.buffer, + file.contents.byteOffset, file.usedBytes); + if (binaryDb.length > 0) { + // All pages, including WAL pages, are in the backup. + // Make the exported copy independent of WAL support. + binaryDb[18] = 1; + binaryDb[19] = 1; + } + return binaryDb; + } finally { + if (backup !== NULL) sqlite3_backup_finish(backup); + if (destination !== NULL) sqlite3_close_v2(destination); + FS.unlink("/" + filename); + } }; /** Close the database, and all associated prepared statements. diff --git a/src/exported_functions.json b/src/exported_functions.json index 3be25955..d61b648a 100644 --- a/src/exported_functions.json +++ b/src/exported_functions.json @@ -2,6 +2,9 @@ "_malloc", "_free", "_sqlite3_open", +"_sqlite3_backup_init", +"_sqlite3_backup_step", +"_sqlite3_backup_finish", "_sqlite3_exec", "_sqlite3_free", "_sqlite3_errmsg", diff --git a/test/test_export.js b/test/test_export.js new file mode 100644 index 00000000..da67b90a --- /dev/null +++ b/test/test_export.js @@ -0,0 +1,237 @@ +exports.test = function(SQL, assert) { + function values(db, query) { + return db.exec(query)[0].values; + } + + // A fresh database is exportable, including inside an empty transaction. + for (const transaction of [false, true]) { + const empty = new SQL.Database(); + let copy; + try { + if (transaction) empty.run('BEGIN'); + const bytes = empty.export(); + assert.ok(bytes instanceof Uint8Array); + copy = new SQL.Database(bytes); + assert.deepStrictEqual(copy.exec('SELECT * FROM sqlite_master'), []); + if (transaction) empty.run('ROLLBACK'); + empty.run('CREATE TABLE after_export(x)'); + } finally { + if (copy) copy.close(); + empty.close(); + } + } + + const db = new SQL.Database(); + let copy; + let cursor; + let iterator; + try { + db.run('PRAGMA page_size = 1024; PRAGMA foreign_keys = ON;' + + 'PRAGMA cache_size = 123; PRAGMA busy_timeout = 456;' + + 'CREATE TABLE items(id INTEGER PRIMARY KEY, data BLOB);' + + "INSERT INTO items VALUES (1, x'0080ff'), (2, NULL);" + + 'CREATE TEMP TABLE scratch(x); INSERT INTO scratch VALUES (7);' + + "ATTACH DATABASE ':memory:' AS aux;" + + 'CREATE TABLE aux.other(x); INSERT INTO aux.other VALUES (9);'); + db.create_function('twice', x => 2 * x); + db.create_aggregate('js_sum', { + init: () => 0, + step: (state, x) => state + x + }); + const calls = []; + db.updateHook((operation, database, table, rowId) => { + calls.push([operation, database, table, rowId]); + }); + cursor = db.prepare('SELECT id FROM items WHERE id >= ? ORDER BY id', [1]); + assert.strictEqual(cursor.step(), true); + assert.deepStrictEqual(cursor.get(), [1]); + iterator = db.iterateStatements('SELECT id FROM items ORDER BY id; SELECT 42'); + const iterated = iterator.next().value; + assert.strictEqual(iterated.step(), true); + assert.deepStrictEqual(iterated.get(), [1]); + const changes = db.getRowsModified(); + const rowId = values(db, 'SELECT last_insert_rowid()'); + + const bytes = db.export(); + const originalBytes = bytes.slice(); + copy = new SQL.Database(bytes); + assert.deepStrictEqual(values(copy, 'SELECT * FROM items ORDER BY id'), [ + [1, new Uint8Array([0, 128, 255])], [2, null] + ], 'The snapshot contains main-database rows and blobs'); + assert.deepStrictEqual(values(copy, 'PRAGMA page_size'), [[1024]]); + assert.throws(() => copy.exec('SELECT * FROM scratch'), /no such table/); + assert.throws(() => copy.exec('SELECT * FROM aux.other'), /no such table/); + + assert.strictEqual(db.getRowsModified(), changes); + assert.deepStrictEqual(values(db, 'SELECT last_insert_rowid()'), rowId); + assert.deepStrictEqual(values(db, 'PRAGMA foreign_keys'), [[1]]); + assert.deepStrictEqual(values(db, 'PRAGMA cache_size'), [[123]]); + assert.deepStrictEqual(values(db, 'PRAGMA busy_timeout'), [[456]]); + assert.deepStrictEqual(values(db, 'SELECT * FROM scratch'), [[7]]); + assert.deepStrictEqual(values(db, 'SELECT * FROM aux.other'), [[9]]); + assert.deepStrictEqual(values(db, 'SELECT twice(js_sum(id)) FROM items'), [[6]], + 'Custom functions and aggregates remain registered'); + assert.deepStrictEqual(values(db, "SELECT reverse('abc')"), [['cba']], + 'Extension functions remain registered'); + + assert.strictEqual(cursor.step(), true); + assert.deepStrictEqual(cursor.get(), [2], 'Cursor position and bindings survive'); + assert.strictEqual(cursor.step(), false); + cursor.free(); + cursor = null; + assert.strictEqual(iterated.step(), true); + assert.deepStrictEqual(iterated.get(), [2], 'Iterator cursor survives'); + const next = iterator.next().value; + assert.strictEqual(next.step(), true); + assert.deepStrictEqual(next.get(), [42], 'Iterator can prepare its next statement'); + assert.strictEqual(iterator.next().done, true); + iterator = null; + + assert.deepStrictEqual(calls, [], 'Export does not report updates'); + db.run('INSERT INTO items VALUES (3, NULL)'); + db.export(); + db.run('INSERT INTO items VALUES (4, NULL)'); + assert.deepStrictEqual(calls, [ + ['insert', 'main', 'items', 3], ['insert', 'main', 'items', 4] + ], 'The hook survives repeated exports'); + db.updateHook(null); + db.run('INSERT INTO items VALUES (5, NULL)'); + assert.strictEqual(calls.length, 2, 'The hook can still be unregistered'); + assert.deepStrictEqual(bytes, originalBytes, + 'Subsequent writes and exports do not change earlier snapshots'); + bytes[0] = 0; + assert.strictEqual(db.export()[0], 83, + 'Mutating an exported array does not change the database'); + } finally { + if (iterator) iterator.return(); + if (cursor) cursor.free(); + if (copy) copy.close(); + db.close(); + } + assert.throws(() => db.export(), /Database closed/); + + // Export fills one result buffer page by page, without recopying the file. + const large = new SQL.Database(); + let largeBytes; + try { + large.run('CREATE TABLE t(data); INSERT INTO t VALUES (zeroblob(3145728))'); + const prototype = Object.getPrototypeOf(Uint8Array.prototype); + const set = prototype.set; + let largestCopy = 0; + prototype.set = function(source, offset) { + largestCopy = Math.max(largestCopy, source.length); + return set.call(this, source, offset); + }; + try { + largeBytes = large.export(); + } finally { + prototype.set = set; + } + assert.ok(largestCopy <= 65536, + 'Export never copies a growing file or a full-size result array'); + assert.strictEqual(largeBytes.buffer.byteLength, largeBytes.byteLength, + 'The result owns an exactly sized buffer, not spare MEMFS capacity'); + } finally { + large.close(); + } + const largeCopy = new SQL.Database(largeBytes); + try { + assert.deepStrictEqual(values(largeCopy, 'PRAGMA integrity_check'), [['ok']]); + assert.deepStrictEqual(values(largeCopy, 'SELECT length(data) FROM t'), + [[3145728]], 'The result remains valid after closing the source'); + } finally { + largeCopy.close(); + } + + for (const pageSize of [512, 4096, 65536]) { + const paged = new SQL.Database(); + let pagedCopy; + try { + paged.run('PRAGMA page_size = ' + pageSize + ';' + + 'CREATE TABLE t(x); INSERT INTO t VALUES (42);'); + const bytes = paged.export(); + assert.strictEqual(bytes.length, + values(paged, 'PRAGMA page_count')[0][0] * pageSize); + assert.strictEqual(bytes.buffer.byteLength, bytes.length); + pagedCopy = new SQL.Database(bytes); + assert.deepStrictEqual(values(pagedCopy, 'SELECT x FROM t'), [[42]]); + } finally { + if (pagedCopy) pagedCopy.close(); + paged.close(); + } + } + + // Write transactions fail promptly without losing writes or savepoints. + const transactional = new SQL.Database(); + let committed; + try { + transactional.run('CREATE TABLE t(x); INSERT INTO t VALUES (1);' + + 'BEGIN; INSERT INTO t VALUES (2);' + + 'SAVEPOINT saved; INSERT INTO t VALUES (3);'); + for (let i = 0; i < 3; i++) { + assert.throws(() => transactional.export(), /busy or locked/); + } + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t ORDER BY x'), + [[1], [2], [3]], 'Failed exports leave uncommitted writes intact'); + transactional.run('ROLLBACK TO saved'); + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t ORDER BY x'), + [[1], [2]], 'Savepoints survive a failed export'); + transactional.run('ROLLBACK'); + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t'), [[1]], + 'The transaction can still be rolled back after a failed export'); + + transactional.run('BEGIN; INSERT INTO t VALUES (4)'); + assert.throws(() => transactional.export(), /busy or locked/); + transactional.run('COMMIT'); + committed = new SQL.Database(transactional.export()); + assert.deepStrictEqual(values(transactional, 'SELECT x FROM t ORDER BY x'), + [[1], [4]], 'A failed export does not prevent commit or a later export'); + assert.deepStrictEqual(values(committed, 'SELECT x FROM t ORDER BY x'), + [[1], [4]]); + transactional.run('BEGIN IMMEDIATE'); + assert.throws(() => transactional.export(), /busy or locked/, + 'A write transaction is rejected even before it changes any rows'); + transactional.run('ROLLBACK; BEGIN'); + values(transactional, 'SELECT x FROM t'); + const readCopy = new SQL.Database(transactional.export()); + try { + assert.deepStrictEqual(values(readCopy, 'SELECT x FROM t ORDER BY x'), + [[1], [4]], 'Active read transactions can be exported'); + } finally { + readCopy.close(); + } + transactional.run('ROLLBACK'); + } finally { + if (committed) committed.close(); + transactional.close(); + } + + // Exclusive locking lets WAL work without shared-memory VFS methods. + const wal = new SQL.Database(); + let walBytes; + try { + wal.run('PRAGMA locking_mode = EXCLUSIVE'); + assert.deepStrictEqual(values(wal, 'PRAGMA journal_mode = WAL'), [['wal']]); + wal.run('PRAGMA wal_autocheckpoint = 0;' + + 'CREATE TABLE t(x); INSERT INTO t VALUES (1);' + + 'BEGIN; INSERT INTO t VALUES (2);'); + assert.throws(() => wal.export(), /busy or locked/); + wal.run('ROLLBACK'); + walBytes = wal.export(); + assert.deepStrictEqual([walBytes[18], walBytes[19]], [1, 1], + 'Exported files do not require WAL support'); + assert.deepStrictEqual(values(wal, 'PRAGMA journal_mode'), [['wal']], + 'The source keeps its journal mode'); + assert.deepStrictEqual(values(wal, 'SELECT x FROM t'), [[1]]); + } finally { + wal.close(); + } + const walCopy = new SQL.Database(walBytes); + try { + assert.deepStrictEqual(values(walCopy, 'SELECT x FROM t ORDER BY x'), + [[1]], 'A WAL snapshot opens independently after closing the source'); + walCopy.run('INSERT INTO t VALUES (3)'); + } finally { + walCopy.close(); + } +}; diff --git a/test/test_function_registration.js b/test/test_function_registration.js index ddf6eb34..d5a806ee 100644 --- a/test/test_function_registration.js +++ b/test/test_function_registration.js @@ -25,7 +25,7 @@ exports.test = function(SQL, assert) { db.create_aggregate('total', sum); db.create_function('total__finalize', () => 8); assert.deepStrictEqual(values('SELECT total(6), total__finalize()'), [[6, 8]], - 'Registrations work after export resets the registry'); + 'Registrations can be replaced after export'); } finally { db.close(); }