perf(index): faster fresh indexing + parallel reference resolution, byte-identical graphs (#1305)
* perf(index): ~34% faster fresh indexing, byte-identical graphs Profiling a fresh init on a medium TS repo (excalidraw, 657 files) showed the main thread as the critical path: per-row SQLite statement calls, repeated import-resolution walks, and per-row FTS trigger firings, with the parse workers ~75% idle behind it. This lands the semantics-preserving tranche of fixes: - Multi-row batched INSERTs (nodes/edges/unresolved refs/name segments) behind cached per-batch-size prepared statements; row order preserved, so rowid-based resolution determinism (#1015) is unchanged. - storeFileBundle: one transaction per file instead of four; nested transaction() calls now flatten (BEGIN-in-BEGIN previously threw, so no caller depended on nested rollback). - Dedicated store-writer thread for the fresh-DB bulk path (bundles applied in file order on a single writer connection; main thread does no DB work during the parse loop). Kill switch: CODEGRAPH_NO_STORE_WORKER=1. - Bulk FTS mode: drop the nodes_fts sync triggers during the bulk load, rebuild once at the end; crash inside the window self-heals on the next open. - Per-context memos for resolveImportPath/findExportedSymbol + a per-file exported-symbol index, invalidated exactly where clearCaches() already resets the resolver's own caches. - Fast-init on completely fresh DBs (journal in memory, no fsync until the index completes; interrupted init re-runs from scratch). Kill switch: CODEGRAPH_NO_FAST_INIT=1. - MaybeYield returns undefined on the not-due path so per-ref yield checks stop paying a promise + microtask hop each. - Parse pool prewarm for bulk indexing; compile-cache enabled at CLI and worker entry points. Excalidraw fresh init: 5.11s -> 3.36s median (n=5, warm cache, M-series). Graph dumps byte-identical across init, re-index, and sync paths; full suite green (2403 passed). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * perf(resolution): parallel reference resolution with canonical admission Fan resolution batches across a pool of read-only worker threads, each hosting a full ReferenceResolver over its own SQLite connection; results are admitted on the main thread in chunk order, so edge insertion order, row cleanup, failure parking, and deferred post-pass queues are exactly the sequence the single-threaded loop produces. Per-ref inputs match the baseline because the sequential path already resolves each batch against the state committed BEFORE that batch. Validated byte-identical on excalidraw (pool forced on) and apache/dubbo (4,048 Java files): dubbo full index 39s -> 19s (2.05x) with identical graph dumps (91,495 nodes / 223,953 edges). The pool only engages when total pending refs clear a threshold (default 150k, CODEGRAPH_PARALLEL_RESOLVE_MIN to tune, CODEGRAPH_NO_PARALLEL_RESOLVE=1 to disable): measured on a ~58k-ref repo the workers' boot CPU contends with resolution on the same cores and makes indexing slower, so small repos keep the sequential path. When fast-init left the DB in memory-journal mode, WAL is restored before resolution only when the pool will run (readers + rollback-journal writers don't mix). Also: sqlite adapter readOnly open support. TreeCursor spine rewrite of the body walker was built, measured neutral on real repos and equal in a 20k-child microbench (web-tree-sitter's namedChild(i) is not quadratic in this binding), and rejected — per-node JS<->WASM marshaling is the floor, which a traversal swap cannot remove. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
co-authored by
Claude Fable 5
Claude Fable 5
parent
246aee8373
commit
5736e24bb6
+202
-15
@@ -245,6 +245,46 @@ export class QueryBuilder {
|
||||
private segmentedNames: Set<string> = new Set();
|
||||
private static readonly MAX_SEGMENTED_NAMES = 65536;
|
||||
|
||||
// Multi-row INSERT statements, cached per (statement kind × row count). The
|
||||
// bulk write path decomposes N rows into a few fixed batch sizes so each
|
||||
// size's statement is prepared once and reused — one .run() binds a whole
|
||||
// chunk instead of one row, which is where the per-call overhead lives.
|
||||
// Row order within and across chunks is the input order, so rowid assignment
|
||||
// (and therefore resolution's insertion-order disambiguation) is identical
|
||||
// to the one-row-per-run path.
|
||||
private batchStmts: Map<string, SqliteStatement> = new Map();
|
||||
private static readonly BATCH_SIZES: readonly number[] = [128, 32, 8, 1];
|
||||
|
||||
/**
|
||||
* Run `rows` through a multi-row `INSERT` built as `head + (tuple,)*n`,
|
||||
* decomposed greedily into the cached batch sizes. Preserves row order.
|
||||
*/
|
||||
private runBatched(kind: string, head: string, tuple: string, rows: unknown[][]): void {
|
||||
if (rows.length === 0) return;
|
||||
let i = 0;
|
||||
for (const size of QueryBuilder.BATCH_SIZES) {
|
||||
while (rows.length - i >= size) {
|
||||
const key = `${kind}:${size}`;
|
||||
let stmt = this.batchStmts.get(key);
|
||||
if (!stmt) {
|
||||
stmt = this.db.prepare(head + new Array(size).fill(tuple).join(','));
|
||||
this.batchStmts.set(key, stmt);
|
||||
}
|
||||
if (size === 1) {
|
||||
stmt.run(...rows[i]!);
|
||||
} else {
|
||||
const params: unknown[] = [];
|
||||
for (let r = 0; r < size; r++) {
|
||||
const row = rows[i + r]!;
|
||||
for (let c = 0; c < row.length; c++) params.push(row[c]);
|
||||
}
|
||||
stmt.run(...params);
|
||||
}
|
||||
i += size;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
constructor(db: SqliteDatabase) {
|
||||
this.db = db;
|
||||
}
|
||||
@@ -351,17 +391,14 @@ export class QueryBuilder {
|
||||
|
||||
/** Write `name`'s segments into name_segment_vocab (idempotent). */
|
||||
private insertNameSegments(name: string): void {
|
||||
if (this.segmentedNames.has(name)) return;
|
||||
if (this.segmentedNames.size >= QueryBuilder.MAX_SEGMENTED_NAMES) this.segmentedNames.clear();
|
||||
this.segmentedNames.add(name);
|
||||
if (!this.stmts.insertNameSegment) {
|
||||
this.stmts.insertNameSegment = this.db.prepare(
|
||||
'INSERT OR IGNORE INTO name_segment_vocab (segment, name) VALUES (?, ?)',
|
||||
);
|
||||
}
|
||||
for (const segment of splitIdentifierSegments(name)) {
|
||||
this.stmts.insertNameSegment.run(segment, name);
|
||||
}
|
||||
const rows: unknown[][] = [];
|
||||
this.collectNameSegmentRows(name, rows);
|
||||
this.runBatched(
|
||||
'insertNameSegments',
|
||||
'INSERT OR IGNORE INTO name_segment_vocab (segment, name) VALUES ',
|
||||
'(?,?)',
|
||||
rows
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -369,12 +406,124 @@ export class QueryBuilder {
|
||||
*/
|
||||
insertNodes(nodes: Node[]): void {
|
||||
this.db.transaction(() => {
|
||||
// Bulk path: same semantics as insertNode() per row (validation, cache
|
||||
// invalidation, segment vocab), but bound as multi-row INSERTs — the
|
||||
// per-.run() call overhead dominates the store phase on full indexes.
|
||||
const rows: unknown[][] = [];
|
||||
const segmentRows: unknown[][] = [];
|
||||
for (const node of nodes) {
|
||||
this.insertNode(node);
|
||||
if (!node.id || !node.kind || !node.name || !node.filePath || !node.language) {
|
||||
console.error('[CodeGraph] Skipping node with missing required fields:', {
|
||||
id: node.id,
|
||||
kind: node.kind,
|
||||
name: node.name,
|
||||
filePath: node.filePath,
|
||||
language: node.language,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
this.nodeCache.delete(node.id);
|
||||
rows.push([
|
||||
node.id,
|
||||
node.kind,
|
||||
node.name,
|
||||
node.qualifiedName ?? node.name,
|
||||
node.filePath,
|
||||
node.language,
|
||||
node.startLine ?? 0,
|
||||
node.endLine ?? 0,
|
||||
node.startColumn ?? 0,
|
||||
node.endColumn ?? 0,
|
||||
node.docstring ?? null,
|
||||
node.signature ?? null,
|
||||
node.visibility ?? null,
|
||||
node.isExported ? 1 : 0,
|
||||
node.isAsync ? 1 : 0,
|
||||
node.isStatic ? 1 : 0,
|
||||
node.isAbstract ? 1 : 0,
|
||||
node.decorators ? JSON.stringify(node.decorators) : null,
|
||||
node.typeParameters ? JSON.stringify(node.typeParameters) : null,
|
||||
node.returnType ?? null,
|
||||
node.updatedAt ?? Date.now(),
|
||||
]);
|
||||
if (this.isSegmentableKind(node.kind)) this.collectNameSegmentRows(node.name, segmentRows);
|
||||
}
|
||||
this.runBatched(
|
||||
'insertNodes',
|
||||
`INSERT OR REPLACE INTO nodes (
|
||||
id, kind, name, qualified_name, file_path, language,
|
||||
start_line, end_line, start_column, end_column,
|
||||
docstring, signature, visibility,
|
||||
is_exported, is_async, is_static, is_abstract,
|
||||
decorators, type_parameters, return_type, updated_at
|
||||
) VALUES `,
|
||||
'(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
|
||||
rows
|
||||
);
|
||||
this.runBatched(
|
||||
'insertNameSegments',
|
||||
'INSERT OR IGNORE INTO name_segment_vocab (segment, name) VALUES ',
|
||||
'(?,?)',
|
||||
segmentRows
|
||||
);
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Store one file's whole extraction bundle — nodes, edges, unresolved refs,
|
||||
* and the file record — in a SINGLE transaction. The bulk-index path calls
|
||||
* this once per file instead of opening one transaction per table (#1015
|
||||
* file-order commit discipline is unchanged: callers still invoke it in file
|
||||
* order, and row order within is input order).
|
||||
*
|
||||
* Edges MUST already be endpoint-filtered by the caller (the store path
|
||||
* filters to the file's own inserted node ids), so the per-file existence
|
||||
* SELECT that insertEdges() pays is skipped here.
|
||||
*/
|
||||
storeFileBundle(bundle: {
|
||||
nodes: Node[];
|
||||
edges: Edge[];
|
||||
refs: UnresolvedReference[];
|
||||
file: FileRecord;
|
||||
}): void {
|
||||
this.db.transaction(() => {
|
||||
this.insertNodes(bundle.nodes);
|
||||
if (bundle.edges.length > 0) {
|
||||
const rows: unknown[][] = [];
|
||||
for (const edge of bundle.edges) {
|
||||
rows.push([
|
||||
edge.source,
|
||||
edge.target,
|
||||
edge.kind,
|
||||
edge.metadata ? JSON.stringify(edge.metadata) : null,
|
||||
edge.line ?? null,
|
||||
edge.column ?? null,
|
||||
edge.provenance ?? null,
|
||||
]);
|
||||
}
|
||||
this.runBatched(
|
||||
'insertEdges',
|
||||
'INSERT OR IGNORE INTO edges (source, target, kind, metadata, line, col, provenance) VALUES ',
|
||||
'(?,?,?,?,?,?,?)',
|
||||
rows
|
||||
);
|
||||
}
|
||||
if (bundle.refs.length > 0) this.insertUnresolvedRefsBatch(bundle.refs);
|
||||
this.upsertFile(bundle.file);
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect (segment, name) rows for a name, honouring the same session-dedupe
|
||||
* semantics as insertNameSegments(). Shared by the bulk write paths.
|
||||
*/
|
||||
private collectNameSegmentRows(name: string, out: unknown[][]): void {
|
||||
if (this.segmentedNames.has(name)) return;
|
||||
if (this.segmentedNames.size >= QueryBuilder.MAX_SEGMENTED_NAMES) this.segmentedNames.clear();
|
||||
this.segmentedNames.add(name);
|
||||
for (const segment of splitIdentifierSegments(name)) out.push([segment, name]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an existing node
|
||||
*/
|
||||
@@ -510,7 +659,14 @@ export class QueryBuilder {
|
||||
/** Insert segments for a batch of names in one transaction (vocab heal path). */
|
||||
insertNameSegmentsBatch(names: string[]): void {
|
||||
this.db.transaction(() => {
|
||||
for (const name of names) this.insertNameSegments(name);
|
||||
const rows: unknown[][] = [];
|
||||
for (const name of names) this.collectNameSegmentRows(name, rows);
|
||||
this.runBatched(
|
||||
'insertNameSegments',
|
||||
'INSERT OR IGNORE INTO name_segment_vocab (segment, name) VALUES ',
|
||||
'(?,?)',
|
||||
rows
|
||||
);
|
||||
})();
|
||||
}
|
||||
|
||||
@@ -1500,12 +1656,27 @@ export class QueryBuilder {
|
||||
}
|
||||
const existingNodeIds = this.getExistingNodeIds([...endpointIds]);
|
||||
|
||||
const rows: unknown[][] = [];
|
||||
for (const edge of edges) {
|
||||
if (!existingNodeIds.has(edge.source) || !existingNodeIds.has(edge.target)) {
|
||||
continue;
|
||||
}
|
||||
this.insertEdge(edge);
|
||||
rows.push([
|
||||
edge.source,
|
||||
edge.target,
|
||||
edge.kind,
|
||||
edge.metadata ? JSON.stringify(edge.metadata) : null,
|
||||
edge.line ?? null,
|
||||
edge.column ?? null,
|
||||
edge.provenance ?? null,
|
||||
]);
|
||||
}
|
||||
this.runBatched(
|
||||
'insertEdges',
|
||||
'INSERT OR IGNORE INTO edges (source, target, kind, metadata, line, col, provenance) VALUES ',
|
||||
'(?,?,?,?,?,?,?)',
|
||||
rows
|
||||
);
|
||||
})();
|
||||
}
|
||||
|
||||
@@ -1789,9 +1960,25 @@ export class QueryBuilder {
|
||||
insertUnresolvedRefsBatch(refs: UnresolvedReference[]): void {
|
||||
if (refs.length === 0) return;
|
||||
const insert = this.db.transaction(() => {
|
||||
const rows: unknown[][] = [];
|
||||
for (const ref of refs) {
|
||||
this.insertUnresolvedRef(ref);
|
||||
rows.push([
|
||||
ref.fromNodeId,
|
||||
ref.referenceName,
|
||||
ref.referenceKind,
|
||||
ref.line,
|
||||
ref.column,
|
||||
ref.candidates ? JSON.stringify(ref.candidates) : null,
|
||||
ref.filePath ?? '',
|
||||
ref.language ?? 'unknown',
|
||||
]);
|
||||
}
|
||||
this.runBatched(
|
||||
'insertUnresolvedRefs',
|
||||
'INSERT INTO unresolved_refs (from_node_id, reference_name, reference_kind, line, col, candidates, file_path, language) VALUES ',
|
||||
'(?,?,?,?,?,?,?,?)',
|
||||
rows
|
||||
);
|
||||
});
|
||||
insert();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user