Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions benchmark/sqlite/sqlite-prepare-select-read.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
'use strict';
const common = require('../common.js');
const sqlite = require('node:sqlite');
const assert = require('assert');

const bench = common.createBenchmark(main, {
n: [20],
rows: [100, 10000],
method: ['all', 'iterate'],
access: ['named', 'keyed'],
});

const keys = ['text_column', 'integer_column', 'real_column', 'other_column'];

function readNamed(row) {
return row.text_column.length + row.integer_column + row.real_column +
row.other_column;
}

function readKeyed(row) {
let sum = 0;
for (let k = 0; k < keys.length; k++) sum += row[keys[k]] === null ? 0 : 1;
return sum;
}

function main(conf) {
const db = new sqlite.DatabaseSync(':memory:');
db.exec(
'CREATE TABLE foo (text_column TEXT, integer_column INTEGER, ' +
'real_column REAL, blob_column BLOB, other_column INTEGER)',
);
const insert = db.prepare('INSERT INTO foo VALUES (?, ?, ?, ?, ?)');
db.exec('BEGIN');
for (let i = 0; i < conf.rows; i++) {
insert.run(`text ${i}`, i, i / 3, Buffer.from('blob'), i * 2);
}
db.exec('COMMIT');

const stmt = db.prepare('SELECT * FROM foo');
const read = conf.access === 'named' ? readNamed : readKeyed;
const n = conf.n * (1e5 / conf.rows);
let sink = 0;

bench.start();
for (let i = 0; i < n; i++) {
for (const row of conf.method === 'all' ? stmt.all() : stmt.iterate()) {
sink += read(row);
}
}
bench.end(n * conf.rows);

assert.ok(sink > 0);
}
121 changes: 79 additions & 42 deletions src/node_sqlite.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3759,6 +3759,7 @@ void Statement::Finalize() {
}

void Statement::InvalidateColumnNameCache() {
cached_row_template_.Reset();
cached_column_names_.clear();
cached_column_names_reprepare_count_ = -1;
}
Expand Down Expand Up @@ -3981,37 +3982,86 @@ MaybeLocal<Name> Statement::ColumnNameToName(const int column) {
.As<Name>();
}

// Populates `keys` with cached column names, rebuilding the cache if the
// statement was re-prepared.
bool Statement::GetCachedColumnNames(LocalVector<Name>* keys) {
Isolate* isolate = env()->isolate();
// Mirrors V8's String::AsArrayIndex(): canonical decimal below 2^32 - 1.
static bool IsArrayIndex(std::string_view name) {
uint32_t index;
const char* end = name.data() + name.size();
auto [ptr, ec] = std::from_chars(name.data(), end, index);
return ec == std::errc() && ptr == end &&
(name.size() == 1 || name[0] != '0') &&
index != std::numeric_limits<uint32_t>::max();
}

// DictionaryTemplate CHECKs that no key is an array index, interns keys as
// Latin-1 rather than UTF-8, and assumes keys are unique.
static bool CanUseRowTemplate(const std::vector<std::string_view>& names) {
std::unordered_set<std::string_view> seen;
for (std::string_view name : names) {
if (!simdutf::validate_ascii(name.data(), name.size()) ||
IsArrayIndex(name) || !seen.insert(name).second) {
return false;
}
}
return true;
}

// Rebuilds the row shape if the statement was re-prepared.
bool Statement::UpdateColumnNameCache() {
const int reprepare_count =
sqlite3_stmt_status(statement_.get(), SQLITE_STMTSTATUS_REPREPARE, false);
if (reprepare_count != cached_column_names_reprepare_count_) {
cached_column_names_.clear();
const int num_cols = sqlite3_column_count(statement_.get());
if (num_cols == 0) {
cached_column_names_reprepare_count_ = reprepare_count;
return true;
}
if (reprepare_count == cached_column_names_reprepare_count_) return true;

Isolate* isolate = env()->isolate();
InvalidateColumnNameCache();
const int num_cols = sqlite3_column_count(statement_.get());
std::vector<std::string_view> names;
names.reserve(num_cols);
for (int i = 0; i < num_cols; ++i) {
const char* name = sqlite3_column_name(statement_.get(), i);
// ColumnNameToName() below reports the error.
if (name == nullptr) break;
names.emplace_back(name);
}

if (names.size() == static_cast<size_t>(num_cols) &&
CanUseRowTemplate(names)) {
cached_row_template_.Reset(
isolate,
DictionaryTemplate::New(isolate, {names.data(), names.size()}));
} else {
cached_column_names_.reserve(num_cols);
for (int i = 0; i < num_cols; ++i) {
Local<Name> key;
if (!ColumnNameToName(i).ToLocal(&key)) {
InvalidateColumnNameCache();
return false;
}
cached_column_names_.emplace_back(Global<Name>(isolate, key));
cached_column_names_.emplace_back(isolate, key);
}
cached_column_names_reprepare_count_ = reprepare_count;
}
cached_column_names_reprepare_count_ = reprepare_count;
return true;
}

keys->reserve(cached_column_names_.size());
MaybeLocal<Object> Statement::NewRowObject(LocalVector<Value>* values) {
Isolate* isolate = env()->isolate();
if (!cached_row_template_.IsEmpty()) {
// NewInstance() may replace entries, so it needs a MaybeLocal copy.
MaybeStackBuffer<MaybeLocal<Value>, 16> maybe_values(values->size());
for (size_t i = 0; i < values->size(); ++i) maybe_values[i] = (*values)[i];
return NewDictionaryInstanceNullProto(env()->context(),
cached_row_template_.Get(isolate),
{maybe_values.out(), values->size()});
}

DCHECK_EQ(cached_column_names_.size(), values->size());
LocalVector<Name> keys(isolate);
keys.reserve(cached_column_names_.size());
for (const auto& name : cached_column_names_) {
keys->emplace_back(name.Get(isolate));
keys.emplace_back(name.Get(isolate));
}
return true;
return Object::New(
isolate, Null(isolate), keys.data(), values->data(), values->size());
}

MaybeLocal<Value> StatementExecutionHelper::ColumnToValue(Environment* env,
Expand Down Expand Up @@ -4056,12 +4106,15 @@ MaybeLocal<Value> StatementExecutionHelper::All(Environment* env,
int num_cols = 0;
LocalVector<Value> rows(isolate);
LocalVector<Value> row_values(isolate);
LocalVector<Name> row_keys(isolate);

SteppingStatementGuard stepping(db, stmt);
while ((r = sqlite3_step(stmt)) == SQLITE_ROW) {
if (num_cols == 0) {
num_cols = sqlite3_column_count(stmt);
// A running statement is only re-prepared before its first row.
if (!return_arrays && !statement->UpdateColumnNameCache()) {
return MaybeLocal<Value>();
}
}

if (ExtractRowValues(env, stmt, num_cols, use_big_ints, &row_values)
Expand All @@ -4074,16 +4127,10 @@ MaybeLocal<Value> StatementExecutionHelper::All(Environment* env,
Array::New(isolate, row_values.data(), row_values.size());
rows.emplace_back(row_array);
} else {
if (row_keys.size() == 0) {
// Reuses the statement's internalized column names instead of
// re-interning them on every call.
if (!statement->GetCachedColumnNames(&row_keys)) {
return MaybeLocal<Value>();
}
Local<Object> row_obj;
if (!statement->NewRowObject(&row_values).ToLocal(&row_obj)) {
return MaybeLocal<Value>();
}
DCHECK_EQ(row_keys.size(), row_values.size());
Local<Object> row_obj = Object::New(
isolate, Null(isolate), row_keys.data(), row_values.data(), num_cols);
rows.emplace_back(row_obj);
}
}
Expand Down Expand Up @@ -4221,16 +4268,10 @@ MaybeLocal<Value> StatementExecutionHelper::Get(Environment* env,
if (return_arrays) {
result = Array::New(isolate, row_values.data(), row_values.size());
} else {
LocalVector<Name> keys(isolate);
// Reuses the statement's internalized column names instead of
// re-interning them on every call.
if (!statement->GetCachedColumnNames(&keys)) {
if (!statement->UpdateColumnNameCache() ||
!statement->NewRowObject(&row_values).ToLocal(&result)) {
return MaybeLocal<Value>();
}

DCHECK_EQ(keys.size(), row_values.size());
result = Object::New(
isolate, Null(isolate), keys.data(), row_values.data(), num_cols);
}

RESET_AND_CHECK(isolate, db, stmt, needs_reset, MaybeLocal<Value>());
Expand Down Expand Up @@ -5031,7 +5072,6 @@ void StatementIterator::Next(const FunctionCallbackInfo<Value>& args) {

int num_cols = sqlite3_column_count(iter->stmt_->statement_.get());
Local<Value> row_value;
LocalVector<Name> row_keys(isolate);
LocalVector<Value> row_values(isolate);

if (ExtractRowValues(env,
Expand All @@ -5046,13 +5086,10 @@ void StatementIterator::Next(const FunctionCallbackInfo<Value>& args) {
if (iter->stmt_->return_arrays_) {
row_value = Array::New(isolate, row_values.data(), row_values.size());
} else {
// Use cached internalized column names to avoid repeated V8 string
// creation and enable hidden class sharing across row objects.
if (!iter->stmt_->GetCachedColumnNames(&row_keys)) return;

DCHECK_EQ(row_keys.size(), row_values.size());
row_value = Object::New(
isolate, Null(isolate), row_keys.data(), row_values.data(), num_cols);
if (!iter->stmt_->UpdateColumnNameCache() ||
!iter->stmt_->NewRowObject(&row_values).ToLocal(&row_value)) {
return;
}
}

MaybeLocal<Value> values[] = {Boolean::New(isolate, false), row_value};
Expand Down
7 changes: 6 additions & 1 deletion src/node_sqlite.h
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,8 @@ class Statement : public BaseObject {
static void Dispose(const v8::FunctionCallbackInfo<v8::Value>& args);
v8::MaybeLocal<v8::Value> ColumnToValue(const int column);
v8::MaybeLocal<v8::Name> ColumnNameToName(const int column);
bool GetCachedColumnNames(v8::LocalVector<v8::Name>* keys);
// Requires a successful UpdateColumnNameCache() for the current execution.
v8::MaybeLocal<v8::Object> NewRowObject(v8::LocalVector<v8::Value>* values);
void Finalize();
bool IsFinalized();

Expand All @@ -415,9 +416,13 @@ class Statement : public BaseObject {
uint64_t reset_generation_ = 0;
std::optional<std::map<std::string, std::string>> bare_named_params_;
inline int ResetStatement();
// Rows are built from the template, or from the names when they cannot be
// template keys. At most one of the two is populated.
v8::Global<v8::DictionaryTemplate> cached_row_template_;
std::vector<v8::Global<v8::Name>> cached_column_names_;
int cached_column_names_reprepare_count_ = -1;
void InvalidateColumnNameCache();
bool UpdateColumnNameCache();
bool BindParams(const v8::FunctionCallbackInfo<v8::Value>& args);
bool BindValue(const v8::Local<v8::Value>& value, const int index);

Expand Down
100 changes: 100 additions & 0 deletions test/parallel/test-sqlite-row-shape.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
// Flags: --allow-natives-syntax
'use strict';
const { skipIfSQLiteMissing } = require('../common');
skipIfSQLiteMissing();
const { DatabaseSync } = require('node:sqlite');
const { suite, test } = require('node:test');

const hasFastProperties = eval('(o) => %HasFastProperties(o)');
const haveSameMap = eval('(a, b) => %HaveSameMap(a, b)');

function assertSharedFastShape(t, rows) {
t.assert.ok(rows.length >= 2);
for (const row of rows) {
t.assert.strictEqual(Object.getPrototypeOf(row), null);
t.assert.ok(hasFastProperties(row));
t.assert.ok(haveSameMap(rows[0], row));
}
}

function createDatabase() {
const db = new DatabaseSync(':memory:');
db.exec('CREATE TABLE t (a INTEGER, b TEXT); ' +
"INSERT INTO t VALUES (1, 'x'), (2, 'y'), (3, 'z');");
return db;
}

suite('result rows share a fast map', () => {
test('all()', (t) => {
using db = createDatabase();
using stmt = db.prepare('SELECT a, b FROM t');
assertSharedFastShape(t, stmt.all());
});

test('get()', (t) => {
using db = createDatabase();
using stmt = db.prepare('SELECT a, b FROM t WHERE a = ?');
assertSharedFastShape(t, [stmt.get(1), stmt.get(2)]);
});

test('iterate()', (t) => {
using db = createDatabase();
using stmt = db.prepare('SELECT a, b FROM t');
assertSharedFastShape(t, [...stmt.iterate()]);
});
});

test('rows with differing value types keep their values', (t) => {
using db = new DatabaseSync(':memory:');
db.exec('CREATE TABLE t (v); ' +
"INSERT INTO t VALUES (NULL), (1), (1.5), ('s'), (x'01');");
using stmt = db.prepare('SELECT v FROM t ORDER BY rowid');
const expected = [
{ __proto__: null, v: null },
{ __proto__: null, v: 1 },
{ __proto__: null, v: 1.5 },
{ __proto__: null, v: 's' },
{ __proto__: null, v: new Uint8Array([1]) },
];
t.assert.deepStrictEqual(stmt.all(), expected);
t.assert.deepStrictEqual([...stmt.iterate()], expected);
stmt.setReadBigInts(true);
t.assert.deepStrictEqual(stmt.all()[1], { __proto__: null, v: 1n });
});

test('column names that cannot be template keys', (t) => {
using db = new DatabaseSync(':memory:');
const cases = [
['SELECT 1 AS "0"', { __proto__: null, 0: 1 }],
['SELECT 1 AS "4294967294"', { __proto__: null, 4294967294: 1 }],
['SELECT 1 AS a, 2 AS a', { __proto__: null, a: 2 }],
['SELECT 1 AS "café"', { __proto__: null, café: 1 }],
];
for (const [sql, row] of cases) {
using stmt = db.prepare(sql);
t.assert.deepStrictEqual(stmt.get(), row);
t.assert.deepStrictEqual(stmt.all(), [row]);
t.assert.deepStrictEqual([...stmt.iterate()], [row]);
}
});

test('names that look numeric but are not array indices', (t) => {
using db = new DatabaseSync(':memory:');
using stmt = db.prepare(
'SELECT 1 AS "01", 2 AS "4294967295", 3 AS "-1", 4 AS "1.5"');
const row = {
__proto__: null, ['01']: 1, 4294967295: 2, ['-1']: 3, ['1.5']: 4,
};
t.assert.deepStrictEqual(stmt.get(), row);
t.assert.deepStrictEqual([...stmt.iterate()], [row]);
});

test('iterate() picks up columns added by a re-prepare', (t) => {
using db = new DatabaseSync(':memory:');
db.exec("CREATE TABLE t (a); INSERT INTO t VALUES ('x');");
using stmt = db.prepare('SELECT * FROM t');
t.assert.deepStrictEqual([...stmt.iterate()], [{ __proto__: null, a: 'x' }]);
db.exec("ALTER TABLE t ADD COLUMN b DEFAULT 'y';");
t.assert.deepStrictEqual([...stmt.iterate()],
[{ __proto__: null, a: 'x', b: 'y' }]);
});
Loading