fix(resolution): stream node-kind scans in synthesis to fix OOM on dense files (#610) (#653)

The callback/observer synthesizers loaded every function and method node into
memory at once (getNodesByKind('function'/'method')) before scanning them down
to a tiny matched subset. On a symbol-dense project that array is gigabytes, so
indexing spiked the JS heap and aborted with "JavaScript heap out of memory".

Add QueryBuilder.iterateNodesByKind (a lazy node:sqlite cursor) and stream the
synthesizer scans instead of materializing them. Parsing and reference
resolution were already bounded; only the synthesis enumeration wasn't.

Measured on 80 files x 14k functions (~1.1M nodes): peak RSS 3717 MB -> 1318 MB,
no OOM. Full suite green; synthesized edges unchanged.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
2026-06-02 14:23:33 -05:00
committed by GitHub
co-authored by Claude Opus 4.8
parent c9559d9991
commit 2a22f9f55a
5 changed files with 109 additions and 10 deletions
+16
View File
@@ -681,6 +681,22 @@ export class QueryBuilder {
return rows.map(rowToNode);
}
/**
* Stream every node of a kind one at a time (lazy) instead of materializing
* them all like {@link getNodesByKind}. For unbounded kinds (`function`,
* `method`) on a symbol-dense project the full array is gigabytes; the
* dynamic-edge synthesizers only scan-and-filter, so they iterate to keep
* memory O(1) in the node count rather than O(nodes) (#610).
*/
*iterateNodesByKind(kind: NodeKind): IterableIterator<Node> {
// Fresh statement per call (not a cached one): an iterator holds an open
// cursor, so a shared statement would conflict across overlapping scans.
const stmt = this.db.prepare('SELECT * FROM nodes WHERE kind = ?');
for (const row of stmt.iterate(kind)) {
yield rowToNode(row as NodeRow);
}
}
/**
* Get all nodes in the database
*/
+10
View File
@@ -14,6 +14,13 @@ export interface SqliteStatement {
run(...params: any[]): { changes: number; lastInsertRowid: number | bigint };
get(...params: any[]): any;
all(...params: any[]): any[];
/**
* Lazily yield result rows one at a time instead of materializing the whole
* set with `all()`. Use for unbounded scans (e.g. every function/method node)
* so memory stays O(1) in the row count rather than O(rows) — see #610, where
* `all()`-ing every symbol on a dense project spiked the heap into an OOM.
*/
iterate(...params: any[]): IterableIterator<any>;
}
export interface SqliteDatabase {
@@ -72,6 +79,9 @@ class NodeSqliteAdapter implements SqliteDatabase {
all(...params: any[]) {
return stmt.all(...params);
},
iterate(...params: any[]) {
return stmt.iterate(...params);
},
};
}