perf(resolution): defer checkpoints and double-buffer persist during resolution, byte-identical graphs (#1320)
Fresh init on a 4,402-file Java repo (dubbo): 18.2s -> 13.5s (-26%), with the resolution phase going 12.6s -> 7.9s (-38%). Graphs verified byte-identical on both the pool path (dubbo) and the sequential path (excalidraw). Two changes: 1. The fastInit+pool path restored WAL for the resolver workers but left wal_autocheckpoint at its default, so the persist loop inline-checkpointed hot pages all phase long (#1231's pathology inside resolution — measured at 58% of resolution wall). Checkpointing is now deferred behind the bounded valve and folded once at maintenance, mirroring the deferWal path. 2. The resolution loop is double-buffered: batch k+1 is prefetched (OFFSET past batch k's still-pending rows, under an explicit ORDER BY rowid) and fanned out across the pool while batch k's ref cleanup runs on the main thread. Batch settle-waits dropped 2572ms -> 117ms. Correctness invariant found by the byte-identical gate and now documented in the loop: batch k+1's resolution READS batch k's edges (resolveMethodOnType walks supertype chains over extends/implements edges that resolution itself inserts), so edges must persist BEFORE the next batch fans out; only the ref cleanup overlaps. Also extends the CODEGRAPH_SYNTH_TIMINGS instrumentation with phase labels (grammar-init, parse-loop, fts-rebuild, resolver-reinit, resolution, callback-synthesis) and pool timings (worker open/resolve, per-batch mode, persist), so the next profile is one env var away. Suite green (2444). Sync path timings unchanged. Pool floor re-validated: forced-on at 40k refs is still net-slower, so the 150k threshold stands. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
1de7e8f8b5
commit
a2f3c31a97
+28
-1
@@ -464,6 +464,9 @@ export class CodeGraph {
|
||||
const deferWal = !fastInit && process.env.CODEGRAPH_NO_WAL_DEFER !== '1' && this.db.getJournalMode() === 'wal';
|
||||
let walValve: WalCheckpointValve | null = null;
|
||||
let priorAutocheckpoint = 1000;
|
||||
// Set when the fastInit+pool path below defers autocheckpointing, so the
|
||||
// finally knows to restore the interval on that path too.
|
||||
let restoreAutocheckpoint = false;
|
||||
if (deferWal) {
|
||||
priorAutocheckpoint = this.db.getWalAutocheckpoint();
|
||||
this.db.setWalAutocheckpoint(0);
|
||||
@@ -503,7 +506,9 @@ export class CodeGraph {
|
||||
freshDb ? { dbPath: this.db.getPath(), fastInit } : null
|
||||
);
|
||||
} finally {
|
||||
const tFts = Date.now();
|
||||
this.db.endBulkNodeLoad();
|
||||
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] fts-rebuild: ${Date.now() - tFts}ms`);
|
||||
}
|
||||
|
||||
// Fold the parse phase's WAL BEFORE the first post-parse reads
|
||||
@@ -521,10 +526,12 @@ export class CodeGraph {
|
||||
// and silently drop themselves. Re-initializing here gives them a
|
||||
// chance to see the actual project before resolution runs.
|
||||
if (result.success && result.filesIndexed > 0) {
|
||||
const tReinit = Date.now();
|
||||
this.resolver.initialize();
|
||||
// Cross-file finalization (e.g. NestJS RouterModule prefixes). Runs
|
||||
// before resolution so updated names show up in subsequent reads.
|
||||
this.resolver.runPostExtract();
|
||||
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] resolver-reinit: ${Date.now() - tReinit}ms`);
|
||||
}
|
||||
|
||||
// Resolve references to create call/import/extends edges
|
||||
@@ -542,6 +549,24 @@ export class CodeGraph {
|
||||
try {
|
||||
this.db.getDb().pragma('synchronous = NORMAL');
|
||||
this.db.getDb().pragma('journal_mode = WAL');
|
||||
// Defer auto-checkpointing for the resolution phase, same
|
||||
// rationale as the deferWal path above: at the default 1000-page
|
||||
// interval, the persist loop's edge inserts + ref deletes make
|
||||
// SQLite re-write hot B-tree pages into the main DB file inline
|
||||
// on the writer over and over (#1231's pathology — measured as
|
||||
// ~58% of the resolution phase on a 255k-ref repo). The valve
|
||||
// bounds WAL growth off-thread; runMaintenance does the final
|
||||
// fold and the finally restores the interval.
|
||||
priorAutocheckpoint = this.db.getWalAutocheckpoint();
|
||||
this.db.setWalAutocheckpoint(0);
|
||||
restoreAutocheckpoint = true;
|
||||
walValve = new WalCheckpointValve(
|
||||
this.db,
|
||||
undefined,
|
||||
undefined,
|
||||
options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
|
||||
);
|
||||
walValve.start();
|
||||
} catch { /* keep current mode; resolution still works sequentially */ }
|
||||
}
|
||||
|
||||
@@ -551,6 +576,7 @@ export class CodeGraph {
|
||||
total: unresolvedCount,
|
||||
});
|
||||
|
||||
const tResolve = Date.now();
|
||||
await this.resolveReferencesBatched(
|
||||
(current, total) => {
|
||||
options.onProgress?.({
|
||||
@@ -567,6 +593,7 @@ export class CodeGraph {
|
||||
});
|
||||
}
|
||||
);
|
||||
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] resolution: ${Date.now() - tResolve}ms`);
|
||||
|
||||
// Second pass: chained calls whose method lives on a supertype the
|
||||
// receiver conforms to (protocol-extension / inherited / default-
|
||||
@@ -658,7 +685,7 @@ export class CodeGraph {
|
||||
// (SQLite replays the WAL on the next open) and the follow-up write
|
||||
// that folds it is the known cost of a failed run.
|
||||
if (walValve) { walValve.stop(); await walValve.drain(); }
|
||||
if (deferWal) {
|
||||
if (deferWal || restoreAutocheckpoint) {
|
||||
try { this.db.setWalAutocheckpoint(priorAutocheckpoint); } catch { /* connection may be closing */ }
|
||||
}
|
||||
if (fastInit) {
|
||||
|
||||
Reference in New Issue
Block a user