fix(db): stop watchdog-killed sessions from leaking the SQLite WAL without bound (#1431) (#1490)

A SIGKILL'd process (the #850 liveness watchdog, OOM, a crash) leaves its WAL
on disk; the next session appends to the same file; and nothing ever truncated
it — PASSIVE checkpoints fold frames but keep the file at its high-water mark,
and the one shrinking path (a clean last-connection close) is exactly what a
killed-daemon world never takes. Observed at 25.6 GB on a 5.46 GB DB, growing
until the disk filled.

- journal_size_limit on every connection: resetting checkpoints now clip the
  WAL back to the cap instead of leaving it at its high-water mark.
- healOversizedWal() fired from every DatabaseConnection.open: off-thread
  PASSIVE fold + TRUNCATE when the leftover WAL exceeds the cap (64 MB,
  CODEGRAPH_WAL_HEAL_MB to override). Single-flight per connection with
  bounded retries — concurrent passes defeat each other (each checkpoint sees
  the other as a busy reader).
- Daemon/direct MCP watchdogs now pass progressPaths (DB + WAL), extending the
  #1231 slow-disk deferral to the long-lived server so a healthy daemon mid
  slow statement isn't SIGKILL'd — fewer kills, fewer leaked WALs.
- codegraph status shows WAL size (human + JSON) and warns when it dwarfs the
  DB; daemon.log lines and the watchdog kill notice now carry ISO timestamps
  so kills can be placed in time.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

src/db/index.ts

@@ -35,6 +35,35 @@ function configureConnection(db: SqliteDatabase): void {
   db.pragma('cache_size = -64000');      // 64 MB page cache
   db.pragma('temp_store = MEMORY');      // temp tables in memory
   db.pragma('mmap_size = 268435456');    // 256 MB memory-mapped I/O
+  // Without a journal_size_limit the -wal file never shrinks below its
+  // high-water mark while a connection lives: checkpoints fold frames back but
+  // leave the file at full size, so one giant deferred-sync WAL stays giant
+  // forever. With the limit set, any checkpoint that resets the WAL truncates
+  // the file back down. Killed-process leftovers are handled separately by
+  // healOversizedWal() at open. (#1431)
+  db.pragma(`journal_size_limit = ${WAL_HEAL_THRESHOLD_BYTES}`);
+}
+
+/**
+ * WAL size past which `healOversizedWal` (run at every `open`) checkpoints and
+ * truncates the file, and to which `journal_size_limit` clips the WAL after any
+ * resetting checkpoint. A SIGKILL'd process (the #850 liveness watchdog, OOM,
+ * crash) can leave an arbitrarily large WAL behind — a whole deferred-sync
+ * run's worth (#1248) — and before #1431 no later session ever shrank it: the
+ * file just grew, killed session after killed session, until the disk filled
+ * (25.6 GB observed). 64 MB is far above anything a healthy open ever sees
+ * (a clean close deletes the WAL) yet small enough to cap the leak.
+ * Override with `CODEGRAPH_WAL_HEAL_MB` (also feeds `journal_size_limit`).
+ */
+export const WAL_HEAL_THRESHOLD_BYTES = resolveWalHealBytes(process.env.CODEGRAPH_WAL_HEAL_MB);
+
+/** Resolve the heal threshold from the env override (MB); invalid ⇒ 64 MB. */
+export function resolveWalHealBytes(envVal: string | undefined): number {
+  if (envVal !== undefined && envVal !== '') {
+    const n = Number(envVal);
+    if (Number.isFinite(n) && n > 0) return Math.floor(n * 1024 * 1024);
+  }
+  return 64 * 1024 * 1024;
 }
 
 /**
@@ -117,6 +146,10 @@ export class DatabaseConnection {
     // nodes_fts is stale. Rebuild + recreate so search stays in sync.
     conn.healBulkNodeLoad();
 
+    // Self-heal a killed session's leftover oversized WAL (#1431) — one
+    // statSync when healthy, off-thread checkpoint+truncate when not.
+    void conn.healOversizedWal();
+
     return conn;
   }
 
@@ -506,6 +539,52 @@ export class DatabaseConnection {
     return this.checkpointWal('TRUNCATE');
   }
 
+  /**
+   * Shrink a leftover oversized WAL (#1431). A SIGKILL'd session — the #850
+   * liveness watchdog, OOM, a crash — leaves its WAL on disk, the next session
+   * appends to the same file, and (pre-#1431) nothing ever truncated it:
+   * PASSIVE checkpoints fold frames but keep the file at its high-water mark,
+   * and the one shrinking path (a clean last-connection close) is exactly what
+   * the killed world never takes. Unbounded growth until the disk fills.
+   *
+   * Called fire-and-forget from every `open()`: cost is one statSync when the
+   * WAL is small (the overwhelmingly common case). Past the threshold it runs
+   * the off-thread PASSIVE fold then TRUNCATE — both on worker connections
+   * with a busy_timeout, so a racing writer degrades this to a no-op that the
+   * next open retries rather than a stall.
+   */
+  async healOversizedWal(): Promise<{ healed: boolean; beforeBytes: number; afterBytes: number }> {
+    const beforeBytes = this.getWalSizeBytes();
+    if (beforeBytes <= WAL_HEAL_THRESHOLD_BYTES) {
+      return { healed: false, beforeBytes, afterBytes: beforeBytes };
+    }
+    // Single-flight: open() fires this fire-and-forget and callers may also
+    // invoke it explicitly. Two concurrent passes DEFEAT each other — each
+    // checkpoint worker sees the other as a busy reader and no-ops — so share
+    // one in-flight pass instead of racing.
+    this.walHeal ??= this.runWalHeal(beforeBytes).finally(() => { this.walHeal = null; });
+    return this.walHeal;
+  }
+
+  private walHeal: Promise<{ healed: boolean; beforeBytes: number; afterBytes: number }> | null = null;
+
+  private async runWalHeal(beforeBytes: number): Promise<{ healed: boolean; beforeBytes: number; afterBytes: number }> {
+    // A racing reader/writer (another session healing the same file, a query
+    // pool warming up) degrades a checkpoint pass to a busy no-op — retry a
+    // few times before leaving the rest to the next open.
+    for (let attempt = 0; attempt < 3; attempt++) {
+      if (attempt > 0) await new Promise((r) => setTimeout(r, 300));
+      await this.checkpointWalPassive();
+      await this.checkpointWalTruncate();
+      if (this.getWalSizeBytes() <= WAL_HEAL_THRESHOLD_BYTES) break;
+    }
+    const afterBytes = this.getWalSizeBytes();
+    if (process.env.CODEGRAPH_WAL_VALVE_DEBUG) {
+      console.error(`[wal-heal] oversized WAL at open: ${Math.round(beforeBytes / (1024 * 1024))}MB -> ${Math.round(afterBytes / (1024 * 1024))}MB`);
+    }
+    return { healed: afterBytes < beforeBytes, beforeBytes, afterBytes };
+  }
+
   private async checkpointWal(mode: 'PASSIVE' | 'TRUNCATE'): Promise<{ busy: number; log: number; checkpointed: number } | null> {
     if (!this.dbPath || this.dbPath === ':memory:') {
       try {

src/db/queries.ts

@@ -2462,6 +2462,7 @@ export class QueryBuilder {
       edgesByKind,
       filesByLanguage,
       dbSizeBytes: 0, // Set by caller using DatabaseConnection.getSize()
+      walSizeBytes: 0, // Set by caller using DatabaseConnection.getWalSizeBytes()
       lastUpdated: Date.now(),
     };
   }