fix(resolution): WAL containment for the pooled superphase — writer backpressure at pool-idle boundaries (#1332)

At kernel scale the pooled resolution/synthesis superphase grew a 22GB WAL
on a 4.6GB DB (cg1212, §7a.1): autocheckpointing is deferred for the run,
and the valve's timer-driven passive checkpoints stay perpetually partial
against the pool's continuous reads — no mechanism ever completed a
backfill, so the WAL accreted the whole phase's write volume, blowing disk
and feeding page-cache pressure into the 8-core/7GB container OOM.

The valve's writer-side backpressure() hard-cap backstop existed but was
wired only into the PARSE orchestrator. Thread it into the resolution batch
loop at the double-buffer's one pool-idle boundary (batch settled, next not
yet fanned out), after the edge-index recreate, and through the synthesis
insert loops. Parked there, the backfill completes; readers re-enter at
SQLite's backfilled mark and the next persist commit wraps the WAL.

Dubbo validation, same build: valve@16MB peak WAL 251MB (floor = the
single-transaction edge-index recreate) vs defaults 914MB; dumps
byte-identical (441,270 rows); wall unchanged (11s). Suite 2,479 green.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
2026-07-17 07:48:34 -05:00
committed by GitHub
co-authored by Claude Fable 5
parent 04ab45c91f
commit 6e52295ceb
4 changed files with 118 additions and 6 deletions
+10 -2
View File
@@ -591,7 +591,8 @@ export class CodeGraph {
current: done,
total: totalPasses,
});
}
},
walValve ? () => walValve!.backpressure() : undefined
);
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] resolution: ${Date.now() - tResolve}ms`);
@@ -1145,7 +1146,13 @@ export class CodeGraph {
*/
async resolveReferencesBatched(
onProgress?: (current: number, total: number) => void,
onSynthesisProgress?: (done: number, total: number) => void
onSynthesisProgress?: (done: number, total: number) => void,
// The WAL valve's writer-side backstop, threaded into the batch loop's
// pool-idle boundaries. Without it the valve's only lever during
// resolution is timer-driven passive checkpoints, which the pool's
// continuous reads keep perpetually partial — the WAL then accretes the
// whole phase's write volume (22GB on a 4.6GB DB at kernel scale).
backpressure?: () => Promise<void> | null
): Promise<ResolutionResult> {
return this.resolver.resolveAndPersistBatched(onProgress, undefined, onSynthesisProgress, {
dbPath: this.db.getPath(),
@@ -1158,6 +1165,7 @@ export class CodeGraph {
begin: () => this.db.beginBulkEdgeLoad(),
end: () => this.db.endBulkEdgeLoad(),
},
backpressure,
});
}
+15 -1
View File
@@ -3553,7 +3553,11 @@ export async function synthesizeCallbackEdges(
// A live resolver pool to fan the independent passes across (structural type
// so this file never imports the pool — resolver-worker imports THIS file).
// Null/omitted → the sequential path, byte-identical to the pool path.
pool?: { runSynthPass(name: string): Promise<{ edges: Edge[]; ms: number }> } | null
pool?: { runSynthPass(name: string): Promise<{ edges: Edge[]; ms: number }> } | null,
// WAL-valve writer backstop (WalCheckpointValve.backpressure), called at
// pool-idle points in the edge-insert loops below — the passes themselves
// only read; every write in this function happens with the pool idle.
backpressure?: () => Promise<void> | null
): Promise<number> {
// Each sub-pass below is a whole-graph scan, and there are ~30 of them, all
// running synchronously on the indexer's main thread. Their AGGREGATE can run
@@ -3618,10 +3622,18 @@ export async function synthesizeCallbackEdges(
// otherwise orphaned from the struct, and goImplementsEdges (next) derives a
// struct's method set from its `contains` edges — so without this it would
// under-count the interfaces a cross-file struct satisfies. (#583)
// Writer-side WAL backstop for the insert loops here (see the param doc):
// one fstat when under the valve's hard cap, a parked full backfill past it.
const foldIfOver = async (): Promise<void> => {
const bp = backpressure?.();
if (bp) await bp;
};
const goMethodContains = has('go') ? await goCrossFileMethodContainsEdges(queries, yieldToLoop) : NONE;
for (let i = 0; i < goMethodContains.length; i += 2000) {
queries.insertEdges(goMethodContains.slice(i, i + 2000));
await yieldToLoop();
await foldIfOver();
}
await yieldToLoop(); __mark('goMethodContains');
@@ -3633,6 +3645,7 @@ export async function synthesizeCallbackEdges(
for (let i = 0; i < goImpl.length; i += 2000) {
queries.insertEdges(goImpl.slice(i, i + 2000));
await yieldToLoop();
await foldIfOver();
}
await yieldToLoop(); __mark('goImplements');
@@ -3720,6 +3733,7 @@ export async function synthesizeCallbackEdges(
for (let i = 0; i < merged.length; i += 2000) {
queries.insertEdges(merged.slice(i, i + 2000));
await yieldToLoop();
await foldIfOver();
}
__mark('insertMergedEdges');
return merged.length + goImpl.length + goMethodContains.length;
+27 -3
View File
@@ -1338,8 +1338,17 @@ export class ReferenceResolver {
// Sequential fallback on any pool failure. CODEGRAPH_NO_PARALLEL_RESOLVE=1
// disables entirely. bulkEdgeLoad hooks (when provided) bracket the batch
// loop with drop/recreate of the non-unique edge indexes on big runs —
// see DatabaseConnection.beginBulkEdgeLoad.
parallel?: { dbPath: string; bulkEdgeLoad?: { begin: () => void; end: () => void | Promise<void> } }
// see DatabaseConnection.beginBulkEdgeLoad. backpressure (when provided)
// is the WAL valve's writer-side backstop (WalCheckpointValve.backpressure):
// called at pool-idle boundaries so a full backfill can actually complete —
// the valve's timer-driven passive passes stay perpetually partial against
// the pool's continuous reads, which is how a kernel-scale resolution grew
// a 22GB WAL on a 4.6GB DB (migration plan §7a.1).
parallel?: {
dbPath: string;
bulkEdgeLoad?: { begin: () => void; end: () => void | Promise<void> };
backpressure?: () => Promise<void> | null;
}
): Promise<ResolutionResult> {
// Resolution runs on the indexer's MAIN thread, and the #850 liveness
// watchdog SIGKILLs a process whose event loop stalls past its window (60s
@@ -1476,6 +1485,15 @@ export class ReferenceResolver {
const result = await settleBatch(inFlight, batch);
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[pool-timing] batch ${inFlight.mode}: ${batch.length} refs in ${Date.now() - tBatch}ms`);
// WAL-valve backstop at the ONE pool-idle boundary of the double-buffer
// (this batch settled, the next not yet fanned out): past the hard cap
// the writer parks for a full backfill here, where the pool's readers
// are all between statements — so the backfill completes, readers
// re-enter at SQLite's backfilled mark, and the next persist commit
// WRAPS the WAL instead of growing it. No-op (one fstat) under the cap.
const bp = parallel?.backpressure?.();
if (bp) await bp;
// Persist in bounded sub-transactions with yields between: a whole
// batch's edge insert / keyed deletes are otherwise one solid
// synchronous span each on a multi-GB index, sitting BETWEEN the
@@ -1585,6 +1603,11 @@ export class ReferenceResolver {
const tIdx = Date.now();
await parallel!.bulkEdgeLoad!.end();
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] edge-index-recreate: ${Date.now() - tIdx}ms`);
// The recreate just wrote every non-unique edge index into the WAL
// (multi-GB at kernel scale) with the pool idle — fold before the
// synthesis passes pin readers against it for minutes.
const bp = parallel?.backpressure?.();
if (bp) await bp;
}
}
@@ -1601,7 +1624,8 @@ export class ReferenceResolver {
this.queries,
this.context,
onSynthesisProgress,
pool
pool,
parallel?.backpressure
);
} catch {
// synthesis is additive and optional; ignore failures