Skip to content

Commit 8f36f5d

Browse files
geekypunkclaude
geekypunk
and
claude
committed
perf: fetch table indexes for a schema in one query, not one per table
`enrichColumnsWithKeyAndIndexMetadata` called `getTableIndexes` once per table inside its loop. On a wide schema that is hundreds of serial round trips, and it re-runs for every caller that misses the `databaseObjects` cache. Measured on a 567-table MySQL connection reached over an SSH tunnel, `GET /api/connections/{id}/objects`: before 152.66s / 196.30s on cache miss (~270ms per table) after 1.09s - 1.50s on cache miss cached 0.03-0.48s (unchanged) Both pre-fix requests were abandoned by the client (nginx 499). Because there is no stampede guard, a second caller arriving during the first sweep starts its own full sweep, so retrying made it worse. The fix mirrors what `loadForeignKeyColumns` already does one line above: fetch the whole schema once and group in memory. Adds `IntrospectionProvider.getAllTableIndexes` with a default implementation that loops the existing per-table method, so a provider that does not override it is unchanged, plus overrides for both shipped providers: - MySQL: one INFORMATION_SCHEMA.STATISTICS query scoped to a single TABLE_SCHEMA, so it stays bounded on a server hosting many databases. - Postgres: the same joins and filters as the per-table query, with `t.relname = ANY(?)` in place of `t.relname = ?`. Matching bare relnames across schemas is deliberate — it is what the per-table path already does for an unqualified name. Behaviour preserved: - Tables with no indexes map to an empty list, so callers can distinguish "no indexes" from "not scanned" without a per-table fallback. - Objects qualified with a schema other than the connection database still take the per-table path, since the bulk query covers a single schema. - A failed bulk fetch logs and falls back rather than dropping index flags. Verification. The Postgres form was diffed against the per-table form on a live database with EXCEPT ALL in both directions: 744 rows each, zero rows differing either way, composite index column order preserved. Both engines were then exercised end to end with index flags still populated (MySQL: 760 columns with a single-column index and 375 composite, of 4588; Postgres: 32 and 2, of 128) and no fallback warnings logged. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013dSdoA8UGq7PVyvEwWjPXM
1 parent 804e713 commit 8f36f5d

4 files changed

Lines changed: 259 additions & 3 deletions

File tree

‎backend/src/main/java/com/dbaagent/provider/api/IntrospectionProvider.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@
44

55
import java.sql.Connection;
66
import java.sql.SQLException;
7+
import java.util.Collection;
8+
import java.util.HashMap;
79
import java.util.List;
10+
import java.util.Locale;
811
import java.util.Map;
912

1013
/**
@@ -48,6 +51,44 @@ public interface IntrospectionProvider {
4851
*/
4952
List<TableIndex> getTableIndexes(Connection connection, String database, String tableName) throws SQLException;
5053

54+
/**
55+
* Get indexes for every table in a schema in one round trip.
56+
*
57+
* <p>The per-table {@link #getTableIndexes} above is a round trip each, which turns
58+
* enrichment of a wide schema into hundreds of serial queries — painful on any link
59+
* with real latency (an SSH tunnel to a replica, say). This mirrors what
60+
* {@link #getForeignKeys} already does for constraints: fetch the whole schema once
61+
* and group in memory.
62+
*
63+
* <p>Keys are lower-cased bare table names. A table that exists but has no indexes
64+
* maps to an empty list, so callers can tell "no indexes" from "not scanned" and do
65+
* not need a per-table fallback for the empty case.
66+
*
67+
* <p>The default implementation just loops {@link #getTableIndexes}, so a provider
68+
* that does not override this behaves exactly as before.
69+
*
70+
* @param connection The database connection
71+
* @param database The database/schema name
72+
* @param tableNames Tables the caller cares about (used only by the default fallback)
73+
* @return Map of lower-cased table name to that table's indexes
74+
* @throws SQLException If a database error occurs
75+
*/
76+
default Map<String, List<TableIndex>> getAllTableIndexes(
77+
Connection connection, String database, Collection<String> tableNames
78+
) throws SQLException {
79+
Map<String, List<TableIndex>> byTable = new HashMap<>();
80+
for (String tableName : tableNames) {
81+
if (tableName == null) {
82+
continue;
83+
}
84+
byTable.put(
85+
tableName.toLowerCase(Locale.ROOT),
86+
getTableIndexes(connection, database, tableName)
87+
);
88+
}
89+
return byTable;
90+
}
91+
5192
/**
5293
* Get table statistics (size, row count, etc.).
5394
* @param connection The database connection

‎backend/src/main/java/com/dbaagent/provider/mysql/MySQLIntrospectionProvider.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,80 @@ public List<ColumnInfo> getTableColumns(Connection connection, String database,
148148
return columns;
149149
}
150150

151+
/**
152+
* One query for the whole schema instead of one per table.
153+
*
154+
* <p>INFORMATION_SCHEMA.STATISTICS is not cheap on MySQL, and paying for it 567 times
155+
* in a row across a tunnel is what made schema enrichment take minutes. Scoped to a
156+
* single TABLE_SCHEMA so this stays bounded on servers hosting many databases.
157+
*/
158+
@Override
159+
public Map<String, List<TableIndex>> getAllTableIndexes(
160+
Connection connection, String database, Collection<String> tableNames
161+
) throws SQLException {
162+
String query = """
163+
SELECT TABLE_NAME, INDEX_NAME, COLUMN_NAME, NON_UNIQUE, INDEX_TYPE, SEQ_IN_INDEX
164+
FROM INFORMATION_SCHEMA.STATISTICS
165+
WHERE TABLE_SCHEMA = ?
166+
ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX
167+
""";
168+
169+
// table -> index name -> index, so multi-column indexes accumulate their columns
170+
// in SEQ_IN_INDEX order the same way the per-table path builds them.
171+
Map<String, Map<String, TableIndex>> byTable = new HashMap<>();
172+
173+
try (PreparedStatement stmt = connection.prepareStatement(query)) {
174+
stmt.setString(1, database);
175+
stmt.setFetchSize(fetchSize);
176+
stmt.setQueryTimeout(queryTimeoutSeconds);
177+
178+
try (ResultSet rs = stmt.executeQuery()) {
179+
while (rs.next()) {
180+
String tableName = rs.getString("TABLE_NAME");
181+
if (tableName == null) {
182+
continue;
183+
}
184+
String indexName = rs.getString("INDEX_NAME");
185+
String columnName = rs.getString("COLUMN_NAME");
186+
boolean nonUnique = rs.getBoolean("NON_UNIQUE");
187+
String indexType = rs.getString("INDEX_TYPE");
188+
189+
Map<String, TableIndex> indexMap =
190+
byTable.computeIfAbsent(tableName.toLowerCase(Locale.ROOT), k -> new LinkedHashMap<>());
191+
192+
TableIndex index = indexMap.get(indexName);
193+
if (index == null) {
194+
index = new TableIndex();
195+
index.setName(indexName);
196+
index.setType(indexType);
197+
index.setUnique(!nonUnique);
198+
index.setPrimary("PRIMARY".equals(indexName));
199+
index.setColumns(new ArrayList<>());
200+
indexMap.put(indexName, index);
201+
}
202+
index.getColumns().add(columnName);
203+
}
204+
}
205+
}
206+
207+
// Tables with no indexes at all must still be present, so callers can tell an
208+
// unindexed table from one this scan never covered.
209+
Map<String, List<TableIndex>> result = new HashMap<>();
210+
for (String tableName : tableNames) {
211+
if (tableName == null) {
212+
continue;
213+
}
214+
// Callers look up by the name they passed in, but STATISTICS returns bare
215+
// TABLE_NAMEs — so match on the bare name and key the result by the original.
216+
String key = tableName.toLowerCase(Locale.ROOT);
217+
int dot = key.lastIndexOf('.');
218+
String bare = dot > 0 ? key.substring(dot + 1) : key;
219+
Map<String, TableIndex> indexMap = byTable.get(bare);
220+
result.put(key, indexMap == null ? new ArrayList<>() : new ArrayList<>(indexMap.values()));
221+
}
222+
return result;
223+
}
224+
151225
@Override
152226
public List<TableIndex> getTableIndexes(Connection connection, String database, String tableName) throws SQLException {
153227
List<TableIndex> indexes = new ArrayList<>();

‎backend/src/main/java/com/dbaagent/provider/postgres/PostgresIntrospectionProvider.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,93 @@ LEFT JOIN (
211211
return columns;
212212
}
213213

214+
/**
215+
* One query for every requested table instead of one per table.
216+
*
217+
* <p>Same joins and filters as the per-table variant below; only the predicate
218+
* changes, from a single relname to an array of them. Matching on bare relname
219+
* across schemas is deliberate — it is exactly what the per-table path does for an
220+
* unqualified name, so this stays behaviour-preserving. Schema-qualified names are
221+
* left to the per-table path by the caller.
222+
*/
223+
@Override
224+
public Map<String, List<TableIndex>> getAllTableIndexes(
225+
Connection connection, String database, Collection<String> tableNames
226+
) throws SQLException {
227+
String query = """
228+
SELECT
229+
t.relname AS table_name,
230+
i.relname AS index_name,
231+
a.attname AS column_name,
232+
ix.indisunique AS is_unique,
233+
ix.indisprimary AS is_primary,
234+
am.amname AS index_type
235+
FROM pg_class t
236+
JOIN pg_namespace n ON n.oid = t.relnamespace
237+
JOIN pg_index ix ON t.oid = ix.indrelid
238+
JOIN pg_class i ON i.oid = ix.indexrelid
239+
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey)
240+
JOIN pg_am am ON i.relam = am.oid
241+
WHERE t.relkind IN ('r', 'p', 'm', 'v')
242+
AND t.relname = ANY(?)
243+
ORDER BY t.relname, i.relname, a.attnum
244+
""";
245+
246+
Map<String, Map<String, TableIndex>> byTable = new HashMap<>();
247+
String[] names = tableNames.stream()
248+
.filter(Objects::nonNull)
249+
.toArray(String[]::new);
250+
251+
try (PreparedStatement stmt = connection.prepareStatement(query)) {
252+
stmt.setArray(1, connection.createArrayOf("text", names));
253+
if (fetchSize > 0) {
254+
stmt.setFetchSize(fetchSize);
255+
}
256+
stmt.setQueryTimeout(queryTimeoutSeconds);
257+
258+
try (ResultSet rs = stmt.executeQuery()) {
259+
while (rs.next()) {
260+
String tableName = rs.getString("table_name");
261+
if (tableName == null) {
262+
continue;
263+
}
264+
String indexName = rs.getString("index_name");
265+
String columnName = rs.getString("column_name");
266+
boolean isUnique = rs.getBoolean("is_unique");
267+
boolean isPrimary = rs.getBoolean("is_primary");
268+
String indexType = rs.getString("index_type");
269+
270+
Map<String, TableIndex> indexMap =
271+
byTable.computeIfAbsent(tableName.toLowerCase(Locale.ROOT), k -> new LinkedHashMap<>());
272+
273+
TableIndex index = indexMap.get(indexName);
274+
if (index == null) {
275+
index = new TableIndex();
276+
index.setName(indexName);
277+
index.setType(indexType);
278+
index.setUnique(isUnique);
279+
index.setPrimary(isPrimary);
280+
index.setColumns(new ArrayList<>());
281+
indexMap.put(indexName, index);
282+
}
283+
index.getColumns().add(columnName);
284+
}
285+
}
286+
}
287+
288+
// Every requested table gets an entry, so an unindexed table is distinguishable
289+
// from one this scan did not cover.
290+
Map<String, List<TableIndex>> result = new HashMap<>();
291+
for (String tableName : names) {
292+
String key = tableName.toLowerCase(Locale.ROOT);
293+
int dot = key.lastIndexOf('.');
294+
String bare = dot > 0 ? key.substring(dot + 1) : key;
295+
Map<String, TableIndex> indexMap = byTable.get(bare);
296+
result.put(key, indexMap == null ? new ArrayList<>() : new ArrayList<>(indexMap.values()));
297+
}
298+
return result;
299+
}
300+
214301
@Override
215302
public List<TableIndex> getTableIndexes(Connection connection, String database, String tableName) throws SQLException {
216303
List<TableIndex> indexes = new ArrayList<>();

‎backend/src/main/java/com/dbaagent/service/QueryExecutorService.java‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,40 @@ private void enrichColumnsWithKeyAndIndexMetadata(
170170
? loadForeignKeyColumns(connection, connRequest.getDatabase(), provider, connectionId)
171171
: Collections.emptySet();
172172

173+
// Indexes for every table up front, in one round trip. Fetching them
174+
// table-by-table inside the loop below meant one query per table — on a
175+
// 567-table schema behind a tunnel that was minutes, and it ran again for
176+
// every caller that missed the cache.
177+
//
178+
// Only unqualified names, or ones qualified with this connection's own
179+
// database, are covered: the bulk query is scoped to a single schema, so a
180+
// `otherdb.orders` object still takes the per-table path below and keeps
181+
// reading from the schema it names.
182+
Map<String, List<TableIndex>> indexesByTable = Collections.emptyMap();
183+
if (connection != null && provider != null) {
184+
List<String> bulkTables = new ArrayList<>();
185+
for (DatabaseObject obj : objects) {
186+
if (isBulkIndexable(obj, connRequest.getDatabase())) {
187+
bulkTables.add(obj.getName());
188+
}
189+
}
190+
if (!bulkTables.isEmpty()) {
191+
try {
192+
indexesByTable = provider.getAllTableIndexes(
193+
connection, connRequest.getDatabase(), bulkTables
194+
);
195+
} catch (Exception e) {
196+
// Fall back to the per-table path rather than losing index flags.
197+
log.warn(
198+
"Bulk index fetch failed for connection {} ({}); falling back to per-table",
199+
connectionId,
200+
e.getMessage()
201+
);
202+
indexesByTable = Collections.emptyMap();
203+
}
204+
}
205+
}
206+
173207
for (DatabaseObject obj : objects) {
174208
if (obj.getColumns() == null || obj.getColumns().isEmpty()) {
175209
continue;
@@ -181,9 +215,13 @@ private void enrichColumnsWithKeyAndIndexMetadata(
181215
&& obj.getType() != null
182216
&& "table".equalsIgnoreCase(obj.getType())) {
183217
try {
184-
List<TableIndex> indexes = provider.getTableIndexes(
185-
connection, connRequest.getDatabase(), obj.getName()
186-
);
218+
String key = obj.getName().toLowerCase(Locale.ROOT);
219+
List<TableIndex> indexes = indexesByTable.get(key);
220+
if (indexes == null) {
221+
indexes = provider.getTableIndexes(
222+
connection, connRequest.getDatabase(), obj.getName()
223+
);
224+
}
187225
applyIndexFlags(obj, indexes);
188226
} catch (Exception e) {
189227
log.debug(
@@ -240,6 +278,22 @@ private Set<String> loadForeignKeyColumns(
240278
return fkColumns;
241279
}
242280

281+
/**
282+
* True when the bulk (single-schema) index fetch can answer for this object.
283+
* A name qualified with a different schema must not be answered from the
284+
* connection database's index list.
285+
*/
286+
private boolean isBulkIndexable(DatabaseObject obj, String database) {
287+
if (obj == null || obj.getName() == null || !"table".equalsIgnoreCase(obj.getType())) {
288+
return false;
289+
}
290+
int dot = obj.getName().lastIndexOf('.');
291+
if (dot <= 0) {
292+
return true;
293+
}
294+
return obj.getName().substring(0, dot).equalsIgnoreCase(database);
295+
}
296+
243297
private void applyIndexFlags(DatabaseObject obj, List<TableIndex> indexes) {
244298
if (indexes == null || indexes.isEmpty()) {
245299
return;

0 commit comments

Comments
 (0)