Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions benchmark/sqlite/sqlite-prepare-select-read.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
'use strict';
const common = require('../common.js');
const sqlite = require('node:sqlite');
const assert = require('assert');

const bench = common.createBenchmark(main, {
n: [1e4],
rows: [1, 100],
method: ['all', 'iterate', 'get'],
returnArrays: [0, 1],
});

function main(conf) {
const db = new sqlite.Database(':memory:');
db.exec(
'CREATE TABLE foo (text_column TEXT, integer_column INTEGER, real_column REAL, blob_column BLOB)',
);
const insert = db.prepare('INSERT INTO foo VALUES (?, ?, ?, ?)');
for (let i = 0; i < conf.rows; i++) {
insert.run(`text ${i}`, i, i / 3, Buffer.from('example blob data'));
}

const stmt = db.prepare(`SELECT * FROM foo LIMIT ${conf.rows}`);
stmt.setReturnArrays(conf.returnArrays === 1);

// Each row is built and then has every column read once.
const read = conf.returnArrays ?
(row) => row[0].length + row[1] + row[2] + row[3].length :
(row) => row.text_column.length + row.integer_column +
row.real_column + row.blob_column.length;

let sum = 0;
bench.start();
for (let i = 0; i < conf.n; i++) {
if (conf.method === 'get') {
sum += read(stmt.get());
} else {
for (const row of stmt[conf.method]()) sum += read(row);
}
}
bench.end(conf.n);

assert.ok(sum > 0);
}
132 changes: 79 additions & 53 deletions src/node_sqlite.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <cstring>
#include <limits>
#include <string>
#include <string_view>

namespace node {
namespace sqlite {
Expand Down Expand Up @@ -3760,6 +3761,7 @@ void Statement::Finalize() {

void Statement::InvalidateColumnNameCache() {
cached_column_names_.clear();
cached_row_template_.Reset();
cached_column_names_reprepare_count_ = -1;
}

Expand Down Expand Up @@ -3981,37 +3983,79 @@ MaybeLocal<Name> Statement::ColumnNameToName(const int column) {
.As<Name>();
}

// Populates `keys` with cached column names, rebuilding the cache if the
// statement was re-prepared.
bool Statement::GetCachedColumnNames(LocalVector<Name>* keys) {
Isolate* isolate = env()->isolate();
// Size of the stack array RowToObject passes to the template.
static constexpr int kMaxRowTemplateColumns = 64;

// Templates read names as Latin-1 and reject indices and duplicates.
static bool CanUseInRowTemplate(Local<Context> context,
Local<Name> key,
std::string_view name,
const std::vector<std::string_view>& seen) {
return simdutf::validate_ascii(name.data(), name.size()) &&
std::find(seen.begin(), seen.end(), name) == seen.end() &&
key->ToArrayIndex(context).IsEmpty();
}

// Fills either cached_row_template_ or cached_column_names_.
bool Statement::UpdateRowCache() {
const int reprepare_count =
sqlite3_stmt_status(statement_.get(), SQLITE_STMTSTATUS_REPREPARE, false);
if (reprepare_count != cached_column_names_reprepare_count_) {
cached_column_names_.clear();
const int num_cols = sqlite3_column_count(statement_.get());
if (num_cols == 0) {
cached_column_names_reprepare_count_ = reprepare_count;
return true;
}
if (reprepare_count == cached_column_names_reprepare_count_) return true;

InvalidateColumnNameCache();
Isolate* isolate = env()->isolate();
Local<Context> context = env()->context();
const int num_cols = sqlite3_column_count(statement_.get());
LocalVector<Name> keys(isolate);
std::vector<std::string_view> names;
keys.reserve(num_cols);
names.reserve(num_cols);
bool use_template = num_cols <= kMaxRowTemplateColumns;
for (int i = 0; i < num_cols; ++i) {
Local<Name> key;
if (!ColumnNameToName(i).ToLocal(&key)) return false;
std::string_view name = sqlite3_column_name(statement_.get(), i);
use_template =
use_template && CanUseInRowTemplate(context, key, name, names);
keys.push_back(key);
names.push_back(name);
}

if (use_template) {
cached_row_template_.Reset(
isolate,
DictionaryTemplate::New(isolate, {names.data(), names.size()}));
} else {
cached_column_names_.reserve(num_cols);
for (int i = 0; i < num_cols; ++i) {
Local<Name> key;
if (!ColumnNameToName(i).ToLocal(&key)) {
InvalidateColumnNameCache();
return false;
}
cached_column_names_.emplace_back(Global<Name>(isolate, key));
for (Local<Name> key : keys) {
cached_column_names_.emplace_back(isolate, key);
}
cached_column_names_reprepare_count_ = reprepare_count;
}
cached_column_names_reprepare_count_ = reprepare_count;
return true;
}

keys->reserve(cached_column_names_.size());
for (const auto& name : cached_column_names_) {
keys->emplace_back(name.Get(isolate));
MaybeLocal<Object> Statement::RowToObject(LocalVector<Value>* values) {
DCHECK_EQ(
sqlite3_stmt_status(statement_.get(), SQLITE_STMTSTATUS_REPREPARE, false),
cached_column_names_reprepare_count_);
Isolate* isolate = env()->isolate();
if (cached_row_template_.IsEmpty()) {
LocalVector<Name> keys(isolate);
keys.reserve(cached_column_names_.size());
for (const auto& name : cached_column_names_) {
keys.emplace_back(name.Get(isolate));
}
DCHECK_EQ(keys.size(), values->size());
return Object::New(
isolate, Null(isolate), keys.data(), values->data(), keys.size());
}
return true;
DCHECK_LE(values->size(), kMaxRowTemplateColumns);
MaybeLocal<Value> template_values[kMaxRowTemplateColumns];
std::copy(values->begin(), values->end(), template_values);
return NewDictionaryInstanceNullProto(env()->context(),
cached_row_template_.Get(isolate),
{template_values, values->size()});
}

MaybeLocal<Value> StatementExecutionHelper::ColumnToValue(Environment* env,
Expand Down Expand Up @@ -4056,12 +4100,14 @@ MaybeLocal<Value> StatementExecutionHelper::All(Environment* env,
int num_cols = 0;
LocalVector<Value> rows(isolate);
LocalVector<Value> row_values(isolate);
LocalVector<Name> row_keys(isolate);

SteppingStatementGuard stepping(db, stmt);
while ((r = sqlite3_step(stmt)) == SQLITE_ROW) {
if (num_cols == 0) {
num_cols = sqlite3_column_count(stmt);
if (!return_arrays && !statement->UpdateRowCache()) {
return MaybeLocal<Value>();
}
}

if (ExtractRowValues(env, stmt, num_cols, use_big_ints, &row_values)
Expand All @@ -4074,16 +4120,10 @@ MaybeLocal<Value> StatementExecutionHelper::All(Environment* env,
Array::New(isolate, row_values.data(), row_values.size());
rows.emplace_back(row_array);
} else {
if (row_keys.size() == 0) {
// Reuses the statement's internalized column names instead of
// re-interning them on every call.
if (!statement->GetCachedColumnNames(&row_keys)) {
return MaybeLocal<Value>();
}
Local<Object> row_obj;
if (!statement->RowToObject(&row_values).ToLocal(&row_obj)) {
return MaybeLocal<Value>();
}
DCHECK_EQ(row_keys.size(), row_values.size());
Local<Object> row_obj = Object::New(
isolate, Null(isolate), row_keys.data(), row_values.data(), num_cols);
rows.emplace_back(row_obj);
}
}
Expand Down Expand Up @@ -4220,17 +4260,9 @@ MaybeLocal<Value> StatementExecutionHelper::Get(Environment* env,
Local<Value> result;
if (return_arrays) {
result = Array::New(isolate, row_values.data(), row_values.size());
} else {
LocalVector<Name> keys(isolate);
// Reuses the statement's internalized column names instead of
// re-interning them on every call.
if (!statement->GetCachedColumnNames(&keys)) {
return MaybeLocal<Value>();
}

DCHECK_EQ(keys.size(), row_values.size());
result = Object::New(
isolate, Null(isolate), keys.data(), row_values.data(), num_cols);
} else if (!statement->UpdateRowCache() ||
!statement->RowToObject(&row_values).ToLocal(&result)) {
return MaybeLocal<Value>();
}

RESET_AND_CHECK(isolate, db, stmt, needs_reset, MaybeLocal<Value>());
Expand Down Expand Up @@ -5031,7 +5063,6 @@ void StatementIterator::Next(const FunctionCallbackInfo<Value>& args) {

int num_cols = sqlite3_column_count(iter->stmt_->statement_.get());
Local<Value> row_value;
LocalVector<Name> row_keys(isolate);
LocalVector<Value> row_values(isolate);

if (ExtractRowValues(env,
Expand All @@ -5045,14 +5076,9 @@ void StatementIterator::Next(const FunctionCallbackInfo<Value>& args) {

if (iter->stmt_->return_arrays_) {
row_value = Array::New(isolate, row_values.data(), row_values.size());
} else {
// Use cached internalized column names to avoid repeated V8 string
// creation and enable hidden class sharing across row objects.
if (!iter->stmt_->GetCachedColumnNames(&row_keys)) return;

DCHECK_EQ(row_keys.size(), row_values.size());
row_value = Object::New(
isolate, Null(isolate), row_keys.data(), row_values.data(), num_cols);
} else if (!iter->stmt_->UpdateRowCache() ||
!iter->stmt_->RowToObject(&row_values).ToLocal(&row_value)) {
return;
}

MaybeLocal<Value> values[] = {Boolean::New(isolate, false), row_value};
Expand Down
6 changes: 5 additions & 1 deletion src/node_sqlite.h
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,9 @@ class Statement : public BaseObject {
static void Dispose(const v8::FunctionCallbackInfo<v8::Value>& args);
v8::MaybeLocal<v8::Value> ColumnToValue(const int column);
v8::MaybeLocal<v8::Name> ColumnNameToName(const int column);
bool GetCachedColumnNames(v8::LocalVector<v8::Name>* keys);
// Call after sqlite3_step(), which may re-prepare, and before RowToObject.
bool UpdateRowCache();
v8::MaybeLocal<v8::Object> RowToObject(v8::LocalVector<v8::Value>* values);
void Finalize();
bool IsFinalized();

Expand All @@ -417,6 +419,8 @@ class Statement : public BaseObject {
inline int ResetStatement();
std::vector<v8::Global<v8::Name>> cached_column_names_;
int cached_column_names_reprepare_count_ = -1;
// Set instead of cached_column_names_ when the names fit a template.
v8::Global<v8::DictionaryTemplate> cached_row_template_;
void InvalidateColumnNameCache();
bool BindParams(const v8::FunctionCallbackInfo<v8::Value>& args);
bool BindValue(const v8::Local<v8::Value>& value, const int index);
Expand Down
79 changes: 79 additions & 0 deletions test/parallel/test-sqlite-statement-row-shape.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// Flags: --allow-natives-syntax
'use strict';
const { skipIfSQLiteMissing } = require('../common');
skipIfSQLiteMissing();
const assert = require('node:assert');
const { Database } = require('node:sqlite');
const { suite, test } = require('node:test');

function assertSharedShape(rows) {
for (const row of rows) {
assert.strictEqual(Object.getPrototypeOf(row), null);
assert(eval('%HasFastProperties(row)'));
assert(eval('%HaveSameMap(row, rows[0])'));
}
}

suite('result row objects', () => {
const db = new Database(':memory:');
db.exec('CREATE TABLE t (a INTEGER, b TEXT, c REAL)');
const insert = db.prepare('INSERT INTO t VALUES (?, ?, ?)');
insert.run(1, 'x', 1.5);
insert.run(2, 'y', 2.5);
insert.run(3, null, null);

test('all() rows share a fast map', () => {
const rows = db.prepare('SELECT * FROM t ORDER BY a').all();
assert.deepStrictEqual(rows, [
{ __proto__: null, a: 1, b: 'x', c: 1.5 },
{ __proto__: null, a: 2, b: 'y', c: 2.5 },
{ __proto__: null, a: 3, b: null, c: null },
]);
assertSharedShape(rows);
});

test('get() rows share a fast map', () => {
const stmt = db.prepare('SELECT * FROM t WHERE a = ?');
assertSharedShape([stmt.get(1), stmt.get(2), stmt.get(3)]);
});

test('iterate() rows share a fast map', () => {
assertSharedShape(db.prepare('SELECT * FROM t ORDER BY a').iterate().toArray());
});

test('a re-prepared statement picks up the new columns', () => {
using db = new Database(':memory:');
db.exec('CREATE TABLE s (a INTEGER)');
db.exec('INSERT INTO s VALUES (1)');
const stmt = db.prepare('SELECT * FROM s');
assert.deepStrictEqual(stmt.get(), { __proto__: null, a: 1 });
db.exec('ALTER TABLE s ADD COLUMN b INTEGER DEFAULT 2');
assert.deepStrictEqual(stmt.get(), { __proto__: null, a: 1, b: 2 });
});

test('array-index column names', () => {
const stmt = db.prepare('SELECT 1 AS "0", 2 AS b');
assert.deepStrictEqual(stmt.get(), { __proto__: null, 0: 1, b: 2 });
assert.deepStrictEqual(stmt.all(), [{ __proto__: null, 0: 1, b: 2 }]);
});

test('duplicate column names keep the last value', () => {
const stmt = db.prepare('SELECT 1 AS a, 2 AS a');
assert.deepStrictEqual(stmt.get(), { __proto__: null, a: 2 });
assert.deepStrictEqual(stmt.iterate().toArray(),
[{ __proto__: null, a: 2 }]);
});

test('non-ASCII column names', () => {
const row = db.prepare('SELECT 1 AS "café", 2 AS "名前"').get();
assert.deepStrictEqual(row, { __proto__: null, café: 1, 名前: 2 });
});

test('rows wider than the template limit', () => {
const cols = Array.from({ length: 100 }, (_, i) => `${i} AS c${i}`);
const row = db.prepare(`SELECT ${cols.join(', ')}`).get();
const expected = { __proto__: null };
for (let i = 0; i < 100; i++) expected[`c${i}`] = i;
assert.deepStrictEqual(row, expected);
});
});
Loading