The callback/observer synthesizers loaded every function and method node into
memory at once (getNodesByKind('function'/'method')) before scanning them down
to a tiny matched subset. On a symbol-dense project that array is gigabytes, so
indexing spiked the JS heap and aborted with "JavaScript heap out of memory".
Add QueryBuilder.iterateNodesByKind (a lazy node:sqlite cursor) and stream the
synthesizer scans instead of materializing them. Parsing and reference
resolution were already bounded; only the synthesis enumeration wasn't.
Measured on 80 files x 14k functions (~1.1M nodes): peak RSS 3717 MB -> 1318 MB,
no OOM. Full suite green; synthesized edges unchanged.
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
c9559d9991
commit
2a22f9f55a
@@ -681,6 +681,22 @@ export class QueryBuilder {
|
||||
return rows.map(rowToNode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream every node of a kind one at a time (lazy) instead of materializing
|
||||
* them all like {@link getNodesByKind}. For unbounded kinds (`function`,
|
||||
* `method`) on a symbol-dense project the full array is gigabytes; the
|
||||
* dynamic-edge synthesizers only scan-and-filter, so they iterate to keep
|
||||
* memory O(1) in the node count rather than O(nodes) (#610).
|
||||
*/
|
||||
*iterateNodesByKind(kind: NodeKind): IterableIterator<Node> {
|
||||
// Fresh statement per call (not a cached one): an iterator holds an open
|
||||
// cursor, so a shared statement would conflict across overlapping scans.
|
||||
const stmt = this.db.prepare('SELECT * FROM nodes WHERE kind = ?');
|
||||
for (const row of stmt.iterate(kind)) {
|
||||
yield rowToNode(row as NodeRow);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nodes in the database
|
||||
*/
|
||||
|
||||
@@ -14,6 +14,13 @@ export interface SqliteStatement {
|
||||
run(...params: any[]): { changes: number; lastInsertRowid: number | bigint };
|
||||
get(...params: any[]): any;
|
||||
all(...params: any[]): any[];
|
||||
/**
|
||||
* Lazily yield result rows one at a time instead of materializing the whole
|
||||
* set with `all()`. Use for unbounded scans (e.g. every function/method node)
|
||||
* so memory stays O(1) in the row count rather than O(rows) — see #610, where
|
||||
* `all()`-ing every symbol on a dense project spiked the heap into an OOM.
|
||||
*/
|
||||
iterate(...params: any[]): IterableIterator<any>;
|
||||
}
|
||||
|
||||
export interface SqliteDatabase {
|
||||
@@ -72,6 +79,9 @@ class NodeSqliteAdapter implements SqliteDatabase {
|
||||
all(...params: any[]) {
|
||||
return stmt.all(...params);
|
||||
},
|
||||
iterate(...params: any[]) {
|
||||
return stmt.iterate(...params);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user