fix(db): stop watchdog-killed sessions from leaking the SQLite WAL without bound (#1431) (#1490)

A SIGKILL'd process (the #850 liveness watchdog, OOM, a crash) leaves its WAL
on disk; the next session appends to the same file; and nothing ever truncated
it — PASSIVE checkpoints fold frames but keep the file at its high-water mark,
and the one shrinking path (a clean last-connection close) is exactly what a
killed-daemon world never takes. Observed at 25.6 GB on a 5.46 GB DB, growing
until the disk filled.

- journal_size_limit on every connection: resetting checkpoints now clip the
  WAL back to the cap instead of leaving it at its high-water mark.
- healOversizedWal() fired from every DatabaseConnection.open: off-thread
  PASSIVE fold + TRUNCATE when the leftover WAL exceeds the cap (64 MB,
  CODEGRAPH_WAL_HEAL_MB to override). Single-flight per connection with
  bounded retries — concurrent passes defeat each other (each checkpoint sees
  the other as a busy reader).
- Daemon/direct MCP watchdogs now pass progressPaths (DB + WAL), extending the
  #1231 slow-disk deferral to the long-lived server so a healthy daemon mid
  slow statement isn't SIGKILL'd — fewer kills, fewer leaked WALs.
- codegraph status shows WAL size (human + JSON) and warns when it dwarfs the
  DB; daemon.log lines and the watchdog kill notice now carry ISO timestamps
  so kills can be placed in time.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

src/bin/codegraph.ts

@@ -961,6 +961,7 @@ program
           nodeCount: stats.nodeCount,
           edgeCount: stats.edgeCount,
           dbSizeBytes: stats.dbSizeBytes,
+          walSizeBytes: stats.walSizeBytes,
           backend,
           journalMode,
           nodesByKind: stats.nodesByKind,
@@ -1017,6 +1018,19 @@ program
       console.log(`  Nodes:     ${formatNumber(stats.nodeCount)}`);
       console.log(`  Edges:     ${formatNumber(stats.edgeCount)}`);
       console.log(`  DB Size:   ${(stats.dbSizeBytes / 1024 / 1024).toFixed(2)} MB`);
+      // Surface the WAL sidecar (#1431): a WAL that dwarfs the DB at rest is
+      // the killed-session leak — invisible before this line, it only showed
+      // up as a mysteriously full disk. open() above already kicked off the
+      // automatic heal for the oversized case.
+      if (stats.walSizeBytes > 0) {
+        const { WAL_HEAL_THRESHOLD_BYTES } = await import('../db/index');
+        const oversized = stats.walSizeBytes > Math.max(WAL_HEAL_THRESHOLD_BYTES, stats.dbSizeBytes);
+        const walLabel = `${(stats.walSizeBytes / 1024 / 1024).toFixed(2)} MB`;
+        console.log(`  WAL Size:  ${oversized ? chalk.yellow(walLabel) : walLabel}`);
+        if (oversized) {
+          warn('The write-ahead log is larger than the database — killed sessions left it behind. It is reclaimed automatically on open; if it persists across runs, another live CodeGraph process is holding it.');
+        }
+      }
       // Surface the active SQLite backend (node:sqlite — Node's built-in real
       // SQLite, full WAL + FTS5, no native build).
       const backendLabel = chalk.green(`node:sqlite ${getGlyphs().dash} built-in (full WAL)`);

src/db/index.ts

@@ -35,6 +35,35 @@ function configureConnection(db: SqliteDatabase): void {
   db.pragma('cache_size = -64000');      // 64 MB page cache
   db.pragma('temp_store = MEMORY');      // temp tables in memory
   db.pragma('mmap_size = 268435456');    // 256 MB memory-mapped I/O
+  // Without a journal_size_limit the -wal file never shrinks below its
+  // high-water mark while a connection lives: checkpoints fold frames back but
+  // leave the file at full size, so one giant deferred-sync WAL stays giant
+  // forever. With the limit set, any checkpoint that resets the WAL truncates
+  // the file back down. Killed-process leftovers are handled separately by
+  // healOversizedWal() at open. (#1431)
+  db.pragma(`journal_size_limit = ${WAL_HEAL_THRESHOLD_BYTES}`);
+}
+
+/**
+ * WAL size past which `healOversizedWal` (run at every `open`) checkpoints and
+ * truncates the file, and to which `journal_size_limit` clips the WAL after any
+ * resetting checkpoint. A SIGKILL'd process (the #850 liveness watchdog, OOM,
+ * crash) can leave an arbitrarily large WAL behind — a whole deferred-sync
+ * run's worth (#1248) — and before #1431 no later session ever shrank it: the
+ * file just grew, killed session after killed session, until the disk filled
+ * (25.6 GB observed). 64 MB is far above anything a healthy open ever sees
+ * (a clean close deletes the WAL) yet small enough to cap the leak.
+ * Override with `CODEGRAPH_WAL_HEAL_MB` (also feeds `journal_size_limit`).
+ */
+export const WAL_HEAL_THRESHOLD_BYTES = resolveWalHealBytes(process.env.CODEGRAPH_WAL_HEAL_MB);
+
+/** Resolve the heal threshold from the env override (MB); invalid ⇒ 64 MB. */
+export function resolveWalHealBytes(envVal: string | undefined): number {
+  if (envVal !== undefined && envVal !== '') {
+    const n = Number(envVal);
+    if (Number.isFinite(n) && n > 0) return Math.floor(n * 1024 * 1024);
+  }
+  return 64 * 1024 * 1024;
 }
 
 /**
@@ -117,6 +146,10 @@ export class DatabaseConnection {
     // nodes_fts is stale. Rebuild + recreate so search stays in sync.
     conn.healBulkNodeLoad();
 
+    // Self-heal a killed session's leftover oversized WAL (#1431) — one
+    // statSync when healthy, off-thread checkpoint+truncate when not.
+    void conn.healOversizedWal();
+
     return conn;
   }
 
@@ -506,6 +539,52 @@ export class DatabaseConnection {
     return this.checkpointWal('TRUNCATE');
   }
 
+  /**
+   * Shrink a leftover oversized WAL (#1431). A SIGKILL'd session — the #850
+   * liveness watchdog, OOM, a crash — leaves its WAL on disk, the next session
+   * appends to the same file, and (pre-#1431) nothing ever truncated it:
+   * PASSIVE checkpoints fold frames but keep the file at its high-water mark,
+   * and the one shrinking path (a clean last-connection close) is exactly what
+   * the killed world never takes. Unbounded growth until the disk fills.
+   *
+   * Called fire-and-forget from every `open()`: cost is one statSync when the
+   * WAL is small (the overwhelmingly common case). Past the threshold it runs
+   * the off-thread PASSIVE fold then TRUNCATE — both on worker connections
+   * with a busy_timeout, so a racing writer degrades this to a no-op that the
+   * next open retries rather than a stall.
+   */
+  async healOversizedWal(): Promise<{ healed: boolean; beforeBytes: number; afterBytes: number }> {
+    const beforeBytes = this.getWalSizeBytes();
+    if (beforeBytes <= WAL_HEAL_THRESHOLD_BYTES) {
+      return { healed: false, beforeBytes, afterBytes: beforeBytes };
+    }
+    // Single-flight: open() fires this fire-and-forget and callers may also
+    // invoke it explicitly. Two concurrent passes DEFEAT each other — each
+    // checkpoint worker sees the other as a busy reader and no-ops — so share
+    // one in-flight pass instead of racing.
+    this.walHeal ??= this.runWalHeal(beforeBytes).finally(() => { this.walHeal = null; });
+    return this.walHeal;
+  }
+
+  private walHeal: Promise<{ healed: boolean; beforeBytes: number; afterBytes: number }> | null = null;
+
+  private async runWalHeal(beforeBytes: number): Promise<{ healed: boolean; beforeBytes: number; afterBytes: number }> {
+    // A racing reader/writer (another session healing the same file, a query
+    // pool warming up) degrades a checkpoint pass to a busy no-op — retry a
+    // few times before leaving the rest to the next open.
+    for (let attempt = 0; attempt < 3; attempt++) {
+      if (attempt > 0) await new Promise((r) => setTimeout(r, 300));
+      await this.checkpointWalPassive();
+      await this.checkpointWalTruncate();
+      if (this.getWalSizeBytes() <= WAL_HEAL_THRESHOLD_BYTES) break;
+    }
+    const afterBytes = this.getWalSizeBytes();
+    if (process.env.CODEGRAPH_WAL_VALVE_DEBUG) {
+      console.error(`[wal-heal] oversized WAL at open: ${Math.round(beforeBytes / (1024 * 1024))}MB -> ${Math.round(afterBytes / (1024 * 1024))}MB`);
+    }
+    return { healed: afterBytes < beforeBytes, beforeBytes, afterBytes };
+  }
+
   private async checkpointWal(mode: 'PASSIVE' | 'TRUNCATE'): Promise<{ busy: number; log: number; checkpointed: number } | null> {
     if (!this.dbPath || this.dbPath === ':memory:') {
       try {

src/db/queries.ts

@@ -2462,6 +2462,7 @@ export class QueryBuilder {
       edgesByKind,
       filesByLanguage,
       dbSizeBytes: 0, // Set by caller using DatabaseConnection.getSize()
+      walSizeBytes: 0, // Set by caller using DatabaseConnection.getWalSizeBytes()
       lastUpdated: Date.now(),
     };
   }

src/index.ts

@@ -1220,6 +1220,7 @@ export class CodeGraph {
   getStats(): GraphStats {
     const stats = this.queries.getStats();
     stats.dbSizeBytes = this.db.getSize();
+    stats.walSizeBytes = this.db.getWalSizeBytes();
     return stats;
   }
 

src/mcp/index.ts

@@ -103,6 +103,47 @@ function daemonInternalSet(): boolean {
   return !!raw && raw !== '0' && raw.toLowerCase() !== 'false';
 }
 
+/**
+ * Prefix every `process.stderr.write` chunk with an ISO-8601 timestamp. Called
+ * once, only when this process becomes the detached daemon — whose stderr is
+ * appended to `.codegraph/daemon.log`. Before #1431 no log line carried a
+ * timestamp, so watchdog kills and restarts could be counted but never placed
+ * in time. (The watchdog child writes its kill notice through its own
+ * inherited fd 2, bypassing this wrapper — it stamps that line itself.)
+ */
+export function timestampStderrLines(): void {
+  const orig = process.stderr.write.bind(process.stderr);
+  process.stderr.write = ((chunk: string | Uint8Array, ...rest: unknown[]) => {
+    return (orig as (...args: unknown[]) => boolean)(stampLogChunk(chunk), ...rest);
+  }) as typeof process.stderr.write;
+}
+
+/** Prepend `[<ISO-8601>] ` to a log chunk; unknown chunk types pass through. */
+export function stampLogChunk(chunk: string | Uint8Array): string | Uint8Array {
+  try {
+    const stamp = `[${new Date().toISOString()}] `;
+    if (typeof chunk === 'string') return stamp + chunk;
+    if (Buffer.isBuffer(chunk)) return Buffer.concat([Buffer.from(stamp), chunk]);
+  } catch { /* stamping is best-effort; never block the write */ }
+  return chunk;
+}
+
+/**
+ * Watchdog `progressPaths` for a server keyed on `root`'s index: the SQLite DB
+ * + its WAL. With these, the #850 liveness watchdog only kills on heartbeat
+ * silence when the DB files are NOT advancing — the same slow-disk deferral
+ * the CLI `index`/`init` path got in #1231. Without it, one >timeout
+ * synchronous statement on a big DB (multi-GB index behind Windows Defender)
+ * SIGKILLs a perfectly healthy daemon — and a daemon SIGKILL'd at the end of
+ * nearly every session is what ratcheted the WAL leak in #1431. A true wedge
+ * still dies: a wedged loop writes nothing, so the files stay still.
+ */
+export function watchdogProgressPaths(root: string | null): { progressPaths?: string[] } {
+  if (!root) return {};
+  const dbPath = path.join(getCodeGraphDir(root), 'codegraph.db');
+  return { progressPaths: [dbPath, `${dbPath}-wal`] };
+}
+
 /**
  * Resolve the project root the daemon machinery should key on. Returns
  * `null` when no `.codegraph/` is reachable from the candidate path — in
@@ -346,7 +387,7 @@ export class MCPServer {
     this.mode = 'direct';
     this.installSignalHandlers();
     this.installPpidWatchdog();
-    this.livenessWatchdog = installMainThreadWatchdog();
+    this.livenessWatchdog = installMainThreadWatchdog(watchdogProgressPaths(resolveDaemonRoot(this.projectPath)));
   }
 
   /**
@@ -359,6 +400,9 @@ export class MCPServer {
    * and reaps itself via client-refcount + idle timeout (see {@link Daemon}).
    */
   private async startDaemonProcess(): Promise<void> {
+    // In daemon mode stderr IS `.codegraph/daemon.log`; stamp every line so
+    // kills/restarts can be placed in time (#1431 — the log was undatable).
+    timestampStderrLines();
     const root = resolveDaemonRoot(this.projectPath) ?? this.projectPath ?? process.cwd();
     for (let attempt = 0; attempt < TAKEOVER_MAX_RETRIES; attempt++) {
       const lock = tryAcquireDaemonLock(root);
@@ -371,7 +415,7 @@ export class MCPServer {
         // The detached daemon has no PPID watchdog or stdin lifeline, so a
         // wedged main thread would pin a core forever (#850). The liveness
         // watchdog is its only recovery path.
-        this.livenessWatchdog = installMainThreadWatchdog();
+        this.livenessWatchdog = installMainThreadWatchdog(watchdogProgressPaths(root));
         return; // the net.Server keeps the process alive
       }
 

src/mcp/liveness-watchdog.ts

@@ -113,7 +113,9 @@ const capMs = Number(process.argv[3]);
 const progressPaths = process.argv.slice(4);
 const secs = Math.round(timeoutMs / 1000);
 function kill(extra) {
-  try { fs.writeSync(2, Buffer.from('[CodeGraph] Main thread unresponsive for ~' + secs + 's' + (extra || '') + ' — killing the wedged process so a fresh one can start (#850). Disable with CODEGRAPH_NO_WATCHDOG=1.\\n')); } catch (e) {}
+  // Timestamped so daemon.log kills can be correlated with anything (#1431) —
+  // computed here at kill time; this child process is never the wedged one.
+  try { fs.writeSync(2, Buffer.from('[' + new Date().toISOString() + '] [CodeGraph] Main thread unresponsive for ~' + secs + 's' + (extra || '') + ' — killing the wedged process so a fresh one can start (#850). Disable with CODEGRAPH_NO_WATCHDOG=1.\\n')); } catch (e) {}
   try { process.kill(parentPid, 'SIGKILL'); } catch (e) {}
   process.exit(0);
 }

src/types.ts

@@ -574,6 +574,10 @@ export interface GraphStats {
   /** Database size in bytes */
   dbSizeBytes: number;
 
+  /** Size of the SQLite `-wal` sidecar in bytes (0 when absent). A WAL far
+   * larger than the DB at rest means killed sessions left it behind (#1431). */
+  walSizeBytes: number;
+
   /** Last update timestamp */
   lastUpdated: number;
 }