From 8c1e821495a8d1b8906aac36ccc2354945c73777 Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Fri, 17 Jul 2026 08:17:33 -0500 Subject: [PATCH] =?UTF-8?q?fix(db):=20WAL=20valve=20=E2=80=94=20TRUNCATE?= =?UTF-8?q?=20at=20parked=20barriers,=20futility=20latch,=20CODEGRAPH=5FWA?= =?UTF-8?q?L=5FVALVE=5FDEBUG=20(#1334)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three §7a.1 run-1 lessons (kernel-scale 2c/6GB: EXIT=137, WAL 22.2GB with the backpressure hook DEPLOYED): 1. TRUNCATE at parked barriers: a completed passive backfill bounds the un-checkpointed backlog but the FILE only stops growing when a commit finds zero readers holding WAL marks — rare while pool workers cycle (dubbo debug baseline: file climbed monotonically through six completed pass-1 backfills). At a parked barrier the no-reader window is guaranteed, so chop the file there with wal_checkpoint(TRUNCATE) (off-thread, 2s busy_timeout — a racing reader degrades it to a no-op). 2. Futility latch: when backfill gives up (pinned reader), parking again at every over-cap boundary burns a 20-pass checkpoint attempt — each a worker thread + fresh connection against a multi-GB DB — per batch. Two consecutive give-ups now disable parking for 60s; a pinned phase degrades to pre-valve behavior instead of OOM-amplifying. 3. CODEGRAPH_WAL_VALVE_DEBUG=1 surfaces valve decisions without the caller's verbose plumbing, and give-up lines print under CODEGRAPH_SYNTH_TIMINGS — run 1 failed silently because give-ups were verbose-gated. Co-authored-by: Claude Fable 5 --- __tests__/wal-deferral.test.ts | 14 +++++++++++ src/db/index.ts | 30 +++++++++++++++++++--- src/db/wal-valve.ts | 46 ++++++++++++++++++++++++++++++++-- 3 files changed, 85 insertions(+), 5 deletions(-) diff --git a/__tests__/wal-deferral.test.ts b/__tests__/wal-deferral.test.ts index 5ac3340..59b6587 100644 --- a/__tests__/wal-deferral.test.ts +++ b/__tests__/wal-deferral.test.ts @@ -360,3 +360,17 @@ describe('resolution-phase WAL backpressure plumbing (§7a.1)', () => { await cg.close(); }); }); + +describe('checkpointWalTruncate (§7a.1 file containment)', () => { + it('chops a fully-backfilled WAL file to zero', async () => { + const db = openDb(); + db.setWalAutocheckpoint(0); + writeRows(db, 400); + expect(db.getWalSizeBytes()).toBeGreaterThan(1024 * 1024); + const res = await db.checkpointWalTruncate(); + expect(res).not.toBeNull(); + expect(res!.busy).toBe(0); + expect(db.getWalSizeBytes()).toBe(0); // the file itself, not just the backlog + db.close(); + }); +}); diff --git a/src/db/index.ts b/src/db/index.ts index efa0939..1b271ec 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -362,9 +362,29 @@ export class DatabaseConnection { * never run inline on the main thread). */ async checkpointWalPassive(): Promise<{ busy: number; log: number; checkpointed: number } | null> { + return this.checkpointWal('PASSIVE'); + } + + /** + * `PRAGMA wal_checkpoint(TRUNCATE)` — same off-thread pattern as PASSIVE, + * but on success the WAL FILE is chopped to zero. A completed passive + * backfill bounds the un-checkpointed backlog, yet the FILE only stops + * growing when a commit finds ZERO readers holding WAL marks — rare while + * pool workers cycle, so at kernel scale a fully-backfilled WAL still + * accreted the phase's whole write volume on disk (§7a.1: 22GB). The valve + * calls this exactly at a parked barrier (writer parked, pool drained, + * backfill complete) where the no-reader condition is guaranteed rather + * than lucky. The worker sets a short busy_timeout so a racing reader + * degrades this to a no-op (busy=1) instead of a stall. + */ + async checkpointWalTruncate(): Promise<{ busy: number; log: number; checkpointed: number } | null> { + return this.checkpointWal('TRUNCATE'); + } + + private async checkpointWal(mode: 'PASSIVE' | 'TRUNCATE'): Promise<{ busy: number; log: number; checkpointed: number } | null> { if (!this.dbPath || this.dbPath === ':memory:') { try { - const row = this.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get() as Record | undefined; + const row = this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get() as Record | undefined; return row ? { busy: Number(row.busy), log: Number(row.log), checkpointed: Number(row.checkpointed) } : null; } catch { return null; @@ -378,7 +398,11 @@ export class DatabaseConnection { try { const { DatabaseSync } = require('node:sqlite'); const db = new DatabaseSync(workerData.dbPath); - try { row = db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get(); } catch {} + const mode = workerData.mode === 'TRUNCATE' ? 'TRUNCATE' : 'PASSIVE'; + try { + if (mode === 'TRUNCATE') db.exec('PRAGMA busy_timeout = 2000'); + row = db.prepare('PRAGMA wal_checkpoint(' + mode + ')').get(); + } catch {} try { db.close(); } catch {} } catch {} parentPort.postMessage({ row }); @@ -391,7 +415,7 @@ export class DatabaseConnection { resolve(row ? { busy: Number(row.busy), log: Number(row.log), checkpointed: Number(row.checkpointed) } : null); }; try { - const worker = new Worker(workerSource, { eval: true, workerData: { dbPath: this.dbPath } }); + const worker = new Worker(workerSource, { eval: true, workerData: { dbPath: this.dbPath, mode } }); worker.once('message', (m: { row?: Record | null }) => { void worker.terminate(); finish(m?.row ?? null); }); worker.once('error', () => { void worker.terminate(); finish(null); }); worker.once('exit', () => finish(null)); diff --git a/src/db/wal-valve.ts b/src/db/wal-valve.ts index b509527..3b9e672 100644 --- a/src/db/wal-valve.ts +++ b/src/db/wal-valve.ts @@ -81,16 +81,37 @@ export class WalCheckpointValve { private readonly softBytes: number; private readonly hardBytes: number; + /** + * Futility latch: consecutive backfill give-ups (a reader pinning the WAL) + * disable further writer pauses for a cooldown, so a pinned phase degrades + * to the pre-valve behavior (unbounded WAL, folded when the pinner exits) + * instead of burning a 20-pass checkpoint attempt — each pass a worker + * thread + fresh connection — at EVERY over-cap boundary. That churn is + * what turned a pinned kernel-scale resolution from slow into OOM-killed + * (§7a.1 run 1: 22GB WAL, exit 137 at an envelope the pre-fix build + * survived). + */ + private consecutiveGiveUps = 0; + private futileUntil = 0; + constructor( private readonly db: DatabaseConnection, softMb: number = resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB), private readonly intervalMs: number = CHECK_INTERVAL_MS, - private readonly log: (msg: string) => void = () => {} + log: (msg: string) => void = () => {} ) { this.softBytes = softMb * 1024 * 1024; this.hardBytes = this.softBytes * HARD_CAP_MULTIPLIER; + // CODEGRAPH_WAL_VALVE_DEBUG=1 surfaces valve decisions to stderr without + // needing the caller's verbose plumbing — the observability gap that let + // §7a.1 run 1 fail silently (give-ups were verbose-gated and invisible). + this.log = process.env.CODEGRAPH_WAL_VALVE_DEBUG + ? (m) => console.error(`[wal-valve] ${m}`) + : log; } + private readonly log: (msg: string) => void; + private mb(n: number): string { return `${Math.round(n / 1024 / 1024)}MB`; } @@ -128,6 +149,7 @@ export class WalCheckpointValve { */ backpressure(): Promise | null { if (this.pause) return this.pause; + if (Date.now() < this.futileUntil) return null; // pinned reader — parking is churn, not progress if (this.growthBytes() <= this.hardBytes) return null; this.log(`backpressure: wal=${this.mb(this.db.getWalSizeBytes())} baseline=${this.mb(this.sizeAtLastFullBackfill)} — pausing writer for full backfill`); const t0 = Date.now(); @@ -176,11 +198,31 @@ export class WalCheckpointValve { if (!res) return; // checkpoint machinery unavailable — don't spin this.log(`backfill pass ${i + 1}: busy=${res.busy} log=${res.log} checkpointed=${res.checkpointed} wal=${this.mb(this.db.getWalSizeBytes())}`); if (res.busy === 0 && res.log === res.checkpointed) { + // Backfill complete AND we are at a parked barrier (backfillFully only + // runs under a writer pause): the no-reader window is guaranteed, so + // chop the FILE too — a fully-backfilled WAL otherwise keeps growing + // whenever commits land while pool readers hold marks (§7a.1: 22GB + // on-disk at kernel scale despite backfills). A racing reader turns + // this into a no-op (busy=1); the passive result above still stands. + const trunc = await this.db.checkpointWalTruncate(); + if (trunc) this.log(`truncate: busy=${trunc.busy} wal=${this.mb(this.db.getWalSizeBytes())}`); this.sizeAtLastFullBackfill = this.db.getWalSizeBytes(); + this.consecutiveGiveUps = 0; + this.futileUntil = 0; return; } } - this.log(`backfill gave up after ${MAX_PAUSED_BACKFILL_PASSES} passes — WAL stays unbounded this cycle`); + this.consecutiveGiveUps++; + if (this.consecutiveGiveUps >= 2) { + this.futileUntil = Date.now() + 60_000; + } + const msg = `backfill gave up after ${MAX_PAUSED_BACKFILL_PASSES} passes (streak ${this.consecutiveGiveUps}${this.futileUntil ? ', parking disabled 60s' : ''}) — a reader is pinning the WAL`; + this.log(msg); + // Give-ups are rare and load-bearing for §7a.1-class diagnosis — surface + // them on any timing-instrumented run, not just valve-debug ones. + if (process.env.CODEGRAPH_SYNTH_TIMINGS && !process.env.CODEGRAPH_WAL_VALVE_DEBUG) { + console.error(`[wal-valve] ${msg}`); + } } private fire(): void {