perf: back up directly into a preallocated export buffer

Author: lovasoa

.github/workflows/CI.yml

@@ -4,6 +4,7 @@ on: [push, pull_request]
 
 permissions:
   contents: read
+  actions: read
 
 jobs:
   build:
@@ -24,6 +25,14 @@ jobs:
         if-no-files-found: error
     - name: test WebAssembly builds and distribution behavior
       run: npm ci && npm test
+    # One-off comparison; remove after recording the optimization results.
+    - name: Compare optimized backup export with both baselines
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        gh run download 40301139618 --repo sql-js/sql.js --name dist --dir "$RUNNER_TEMP/serialize-dist"
+        gh run download 40304818305 --repo sql-js/sql.js --name dist --dir "$RUNNER_TEMP/backup-before-dist"
+        node test/compare_exports.cjs "$RUNNER_TEMP/serialize-dist" dist "$RUNNER_TEMP/backup-before-dist"
     - name: generate documentation
       run: npm run doc
     - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2

src/api.js

@@ -1144,6 +1144,23 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
         var destination = NULL;
         var backup = NULL;
         try {
+            var file = FS.lookupPath(filename).node;
+            var binaryDb = null;
+            var write = file.stream_ops.write;
+            file.stream_ops = Object.create(file.stream_ops);
+            file.stream_ops.write = function writeExport(
+                stream, buffer, offset, length, position
+            ) {
+                if (binaryDb === null) {
+                    return write(stream, buffer, offset, length, position);
+                }
+                // Write directly into the final JS-owned result buffer.
+                // MEMFS would otherwise replace it on the first write.
+                file.contents.set(buffer.subarray(offset, offset + length),
+                    position);
+                file.usedBytes = Math.max(file.usedBytes, position + length);
+                return length;
+            };
             var rc = sqlite3_open(filename, apiTemp);
             destination = getValue(apiTemp, "i32");
             if (rc !== SQLITE_OK) {
@@ -1153,7 +1170,18 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
             if (backup === NULL) {
                 throw new Error(sqlite3_errmsg(destination));
             }
-            rc = sqlite3_backup_step(backup, -1);
+            // Initialize the backup and reject writers before allocating.
+            rc = sqlite3_backup_step(backup, 0);
+            if (rc === SQLITE_OK) {
+                var sizes = this.exec(
+                    "PRAGMA main.page_count; PRAGMA main.page_size"
+                );
+                binaryDb = new Uint8Array(
+                    sizes[0].values[0][0] * sizes[1].values[0][0]
+                );
+                file.contents = binaryDb;
+                rc = sqlite3_backup_step(backup, -1);
+            }
             // Do not retry: this synchronous call cannot let a writer progress.
             if (rc === SQLITE_BUSY || rc === SQLITE_LOCKED) {
                 throw new Error("Unable to export: SQLite is busy or locked. "
@@ -1172,7 +1200,9 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
                 throw new Error(sqlite3_errmsg(destination));
             }
             destination = NULL;
-            var binaryDb = FS.readFile(filename, { encoding: "binary" });
+            // The private file is closed and unlinked below; hand its buffer
+            // to the caller rather than making a full-size FS.readFile copy.
+            binaryDb = file.contents.subarray(0, file.usedBytes);
             if (binaryDb.length > 0) {
                 // All pages, including WAL pages, are in the backup.
                 // Make the exported copy independent of WAL support.

test/compare_exports.cjs

@@ -1,4 +1,5 @@
-// CI-only comparison: node test/compare_exports.cjs SERIALIZE_DIST BACKUP_DIST
+// CI-only comparison:
+// node test/compare_exports.cjs SERIALIZE_DIST BACKUP_DIST [PREVIOUS_BACKUP_DIST]
 // Each measurement uses a fresh child process; both implementations run on
 // the same machine. Timing is informational, never a pass/fail threshold.
 const assert = require('node:assert/strict');
@@ -95,7 +96,7 @@ async function worker(strategy, directory, mode, rows) {
                 let bytes;
                 let error;
                 try { bytes = db.export(); } catch (e) { error = e; }
-                if (strategy === 'backup' && name !== 'read_transaction') {
+                if (strategy !== 'serialize' && name !== 'read_transaction') {
                     assert.match(error && error.message, /busy or locked/);
                     outcomes[name] = 'rejected; source transaction intact';
                 } else {
@@ -217,6 +218,10 @@ function compare() {
         serialize: path.resolve(process.argv[2]),
         backup: path.resolve(process.argv[3])
     };
+    if (process.argv[4]) {
+        directories.backup_before = path.resolve(process.argv[4]);
+    }
+    const strategies = Object.keys(directories);
     const behavior = {};
     const constrainedMemory = {};
     const bundle = {};
@@ -231,9 +236,9 @@ function compare() {
     }
     const measurements = [];
     for (const rows of [256, 4096, 16384, 32768, 131072]) {
-        const trials = { serialize: [], backup: [] };
+        const trials = Object.fromEntries(strategies.map(strategy => [strategy, []]));
         for (let repeat = 0; repeat < 3; repeat++) {
-            const order = repeat % 2 ? ['backup', 'serialize'] : ['serialize', 'backup'];
+            const order = repeat % 2 ? strategies.slice().reverse() : strategies;
             for (const strategy of order) {
                 trials[strategy].push(child(strategy, directories[strategy], 'timing', rows));
             }

test/test_export.js

@@ -110,6 +110,56 @@ exports.test = function(SQL, assert) {
     }
     assert.throws(() => db.export(), /Database closed/);
 
+    // Export fills one result buffer page by page, without recopying the file.
+    const large = new SQL.Database();
+    let largeBytes;
+    try {
+        large.run('CREATE TABLE t(data); INSERT INTO t VALUES (zeroblob(3145728))');
+        const set = Uint8Array.prototype.set;
+        let largestCopy = 0;
+        Uint8Array.prototype.set = function(source, offset) {
+            largestCopy = Math.max(largestCopy, source.length);
+            return set.call(this, source, offset);
+        };
+        try {
+            largeBytes = large.export();
+        } finally {
+            Uint8Array.prototype.set = set;
+        }
+        assert.ok(largestCopy <= 65536,
+            'Export never copies a growing file or a full-size result array');
+        assert.strictEqual(largeBytes.buffer.byteLength, largeBytes.byteLength,
+            'The result owns an exactly sized buffer, not spare MEMFS capacity');
+    } finally {
+        large.close();
+    }
+    const largeCopy = new SQL.Database(largeBytes);
+    try {
+        assert.deepStrictEqual(values(largeCopy, 'PRAGMA integrity_check'), [['ok']]);
+        assert.deepStrictEqual(values(largeCopy, 'SELECT length(data) FROM t'),
+            [[3145728]], 'The result remains valid after closing the source');
+    } finally {
+        largeCopy.close();
+    }
+
+    for (const pageSize of [512, 4096, 65536]) {
+        const paged = new SQL.Database();
+        let pagedCopy;
+        try {
+            paged.run('PRAGMA page_size = ' + pageSize + ';'
+                + 'CREATE TABLE t(x); INSERT INTO t VALUES (42);');
+            const bytes = paged.export();
+            assert.strictEqual(bytes.length,
+                values(paged, 'PRAGMA page_count')[0][0] * pageSize);
+            assert.strictEqual(bytes.buffer.byteLength, bytes.length);
+            pagedCopy = new SQL.Database(bytes);
+            assert.deepStrictEqual(values(pagedCopy, 'SELECT x FROM t'), [[42]]);
+        } finally {
+            if (pagedCopy) pagedCopy.close();
+            paged.close();
+        }
+    }
+
     // Write transactions fail promptly without losing writes or savepoints.
     const transactional = new SQL.Database();
     let committed;