fix(db): WAL valve — TRUNCATE at parked barriers, futility latch, CODEGRAPH_WAL_VALVE_DEBUG (#1334)

Three §7a.1 run-1 lessons (kernel-scale 2c/6GB: EXIT=137, WAL 22.2GB with
the backpressure hook DEPLOYED):

1. TRUNCATE at parked barriers: a completed passive backfill bounds the
   un-checkpointed backlog but the FILE only stops growing when a commit
   finds zero readers holding WAL marks — rare while pool workers cycle
   (dubbo debug baseline: file climbed monotonically through six completed
   pass-1 backfills). At a parked barrier the no-reader window is
   guaranteed, so chop the file there with wal_checkpoint(TRUNCATE)
   (off-thread, 2s busy_timeout — a racing reader degrades it to a no-op).

2. Futility latch: when backfill gives up (pinned reader), parking again at
   every over-cap boundary burns a 20-pass checkpoint attempt — each a
   worker thread + fresh connection against a multi-GB DB — per batch. Two
   consecutive give-ups now disable parking for 60s; a pinned phase degrades
   to pre-valve behavior instead of OOM-amplifying.

3. CODEGRAPH_WAL_VALVE_DEBUG=1 surfaces valve decisions without the
   caller's verbose plumbing, and give-up lines print under
   CODEGRAPH_SYNTH_TIMINGS — run 1 failed silently because give-ups were
   verbose-gated.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

src/db/index.ts

@@ -362,9 +362,29 @@ export class DatabaseConnection {
    * never run inline on the main thread).
    */
   async checkpointWalPassive(): Promise<{ busy: number; log: number; checkpointed: number } | null> {
+    return this.checkpointWal('PASSIVE');
+  }
+
+  /**
+   * `PRAGMA wal_checkpoint(TRUNCATE)` — same off-thread pattern as PASSIVE,
+   * but on success the WAL FILE is chopped to zero. A completed passive
+   * backfill bounds the un-checkpointed backlog, yet the FILE only stops
+   * growing when a commit finds ZERO readers holding WAL marks — rare while
+   * pool workers cycle, so at kernel scale a fully-backfilled WAL still
+   * accreted the phase's whole write volume on disk (§7a.1: 22GB). The valve
+   * calls this exactly at a parked barrier (writer parked, pool drained,
+   * backfill complete) where the no-reader condition is guaranteed rather
+   * than lucky. The worker sets a short busy_timeout so a racing reader
+   * degrades this to a no-op (busy=1) instead of a stall.
+   */
+  async checkpointWalTruncate(): Promise<{ busy: number; log: number; checkpointed: number } | null> {
+    return this.checkpointWal('TRUNCATE');
+  }
+
+  private async checkpointWal(mode: 'PASSIVE' | 'TRUNCATE'): Promise<{ busy: number; log: number; checkpointed: number } | null> {
     if (!this.dbPath || this.dbPath === ':memory:') {
       try {
-        const row = this.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get() as Record<string, number> | undefined;
+        const row = this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get() as Record<string, number> | undefined;
         return row ? { busy: Number(row.busy), log: Number(row.log), checkpointed: Number(row.checkpointed) } : null;
       } catch {
         return null;
@@ -378,7 +398,11 @@ export class DatabaseConnection {
         try {
           const { DatabaseSync } = require('node:sqlite');
           const db = new DatabaseSync(workerData.dbPath);
-          try { row = db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get(); } catch {}
+          const mode = workerData.mode === 'TRUNCATE' ? 'TRUNCATE' : 'PASSIVE';
+          try {
+            if (mode === 'TRUNCATE') db.exec('PRAGMA busy_timeout = 2000');
+            row = db.prepare('PRAGMA wal_checkpoint(' + mode + ')').get();
+          } catch {}
           try { db.close(); } catch {}
         } catch {}
         parentPort.postMessage({ row });
@@ -391,7 +415,7 @@ export class DatabaseConnection {
           resolve(row ? { busy: Number(row.busy), log: Number(row.log), checkpointed: Number(row.checkpointed) } : null);
         };
         try {
-          const worker = new Worker(workerSource, { eval: true, workerData: { dbPath: this.dbPath } });
+          const worker = new Worker(workerSource, { eval: true, workerData: { dbPath: this.dbPath, mode } });
           worker.once('message', (m: { row?: Record<string, number> | null }) => { void worker.terminate(); finish(m?.row ?? null); });
           worker.once('error', () => { void worker.terminate(); finish(null); });
           worker.once('exit', () => finish(null));

src/db/wal-valve.ts

@@ -81,16 +81,37 @@ export class WalCheckpointValve {
   private readonly softBytes: number;
   private readonly hardBytes: number;
 
+  /**
+   * Futility latch: consecutive backfill give-ups (a reader pinning the WAL)
+   * disable further writer pauses for a cooldown, so a pinned phase degrades
+   * to the pre-valve behavior (unbounded WAL, folded when the pinner exits)
+   * instead of burning a 20-pass checkpoint attempt — each pass a worker
+   * thread + fresh connection — at EVERY over-cap boundary. That churn is
+   * what turned a pinned kernel-scale resolution from slow into OOM-killed
+   * (§7a.1 run 1: 22GB WAL, exit 137 at an envelope the pre-fix build
+   * survived).
+   */
+  private consecutiveGiveUps = 0;
+  private futileUntil = 0;
+
   constructor(
     private readonly db: DatabaseConnection,
     softMb: number = resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB),
     private readonly intervalMs: number = CHECK_INTERVAL_MS,
-    private readonly log: (msg: string) => void = () => {}
+    log: (msg: string) => void = () => {}
   ) {
     this.softBytes = softMb * 1024 * 1024;
     this.hardBytes = this.softBytes * HARD_CAP_MULTIPLIER;
+    // CODEGRAPH_WAL_VALVE_DEBUG=1 surfaces valve decisions to stderr without
+    // needing the caller's verbose plumbing — the observability gap that let
+    // §7a.1 run 1 fail silently (give-ups were verbose-gated and invisible).
+    this.log = process.env.CODEGRAPH_WAL_VALVE_DEBUG
+      ? (m) => console.error(`[wal-valve] ${m}`)
+      : log;
   }
 
+  private readonly log: (msg: string) => void;
+
   private mb(n: number): string {
     return `${Math.round(n / 1024 / 1024)}MB`;
   }
@@ -128,6 +149,7 @@ export class WalCheckpointValve {
    */
   backpressure(): Promise<void> | null {
     if (this.pause) return this.pause;
+    if (Date.now() < this.futileUntil) return null; // pinned reader — parking is churn, not progress
     if (this.growthBytes() <= this.hardBytes) return null;
     this.log(`backpressure: wal=${this.mb(this.db.getWalSizeBytes())} baseline=${this.mb(this.sizeAtLastFullBackfill)} — pausing writer for full backfill`);
     const t0 = Date.now();
@@ -176,11 +198,31 @@ export class WalCheckpointValve {
       if (!res) return; // checkpoint machinery unavailable — don't spin
       this.log(`backfill pass ${i + 1}: busy=${res.busy} log=${res.log} checkpointed=${res.checkpointed} wal=${this.mb(this.db.getWalSizeBytes())}`);
       if (res.busy === 0 && res.log === res.checkpointed) {
+        // Backfill complete AND we are at a parked barrier (backfillFully only
+        // runs under a writer pause): the no-reader window is guaranteed, so
+        // chop the FILE too — a fully-backfilled WAL otherwise keeps growing
+        // whenever commits land while pool readers hold marks (§7a.1: 22GB
+        // on-disk at kernel scale despite backfills). A racing reader turns
+        // this into a no-op (busy=1); the passive result above still stands.
+        const trunc = await this.db.checkpointWalTruncate();
+        if (trunc) this.log(`truncate: busy=${trunc.busy} wal=${this.mb(this.db.getWalSizeBytes())}`);
         this.sizeAtLastFullBackfill = this.db.getWalSizeBytes();
+        this.consecutiveGiveUps = 0;
+        this.futileUntil = 0;
         return;
       }
     }
-    this.log(`backfill gave up after ${MAX_PAUSED_BACKFILL_PASSES} passes — WAL stays unbounded this cycle`);
+    this.consecutiveGiveUps++;
+    if (this.consecutiveGiveUps >= 2) {
+      this.futileUntil = Date.now() + 60_000;
+    }
+    const msg = `backfill gave up after ${MAX_PAUSED_BACKFILL_PASSES} passes (streak ${this.consecutiveGiveUps}${this.futileUntil ? ', parking disabled 60s' : ''}) — a reader is pinning the WAL`;
+    this.log(msg);
+    // Give-ups are rare and load-bearing for §7a.1-class diagnosis — surface
+    // them on any timing-instrumented run, not just valve-debug ones.
+    if (process.env.CODEGRAPH_SYNTH_TIMINGS && !process.env.CODEGRAPH_WAL_VALVE_DEBUG) {
+      console.error(`[wal-valve] ${msg}`);
+    }
   }
 
   private fire(): void {