codegraph/src/mcp/liveness-watchdog.ts

/**
 * Main-thread liveness watchdog — belt-and-suspenders for #850.
 *
 * The #850 fix removes the one *known* trigger (the uncaught-exception handler
 * no longer formats a raw Error's `.stack` — the step that could enter a
 * non-terminating V8 source-position loop). But ANY synchronous, non-yielding
 * loop on the main thread — a future V8 stack-format pathology, a runaway
 * regex, an accidental `while (true)` — wedges the event loop, and from JS you
 * cannot interrupt it: timers, signal handlers, and the PPID watchdog all run
 * *on* that blocked loop, so the process pins a core forever with no
 * self-recovery (the exact unrecoverable state #850 reported).
 *
 * The only observer still running when the main thread is wedged is another
 * THREAD. This installs a tiny worker thread that watches a heartbeat the main
 * thread bumps through shared memory. If the heartbeat stops advancing across
 * enough consecutive checks (~`timeoutMs` of real time), the worker concludes
 * the main thread is wedged and SIGKILLs the process — the one signal a wedged
 * event loop can't swallow — so a fresh daemon starts on the next connection
 * instead of a zombie pinning a core.
 *
 * **Why count checks, not elapsed wall-clock.** A laptop that sleeps freezes
 * both threads; on wake `Date.now()` has jumped hours but the heartbeat sat
 * still — a wall-clock delta would false-positive and kill a perfectly healthy
 * daemon. Counting *consecutive worker iterations* with no progress is immune:
 * a healthy main thread resumes and bumps the heartbeat within one interval of
 * waking, resetting the count; only a thread that never resumes keeps it
 * climbing. {@link stepHeartbeat} is the pure reducer behind both the worker
 * and the unit tests.
 *
 * **Why it won't fire on real work.** Heavy parsing runs in the parse worker
 * (off this thread) and indexing shells out to a child process, so the daemon's
 * main thread only ever does fast, bounded work (socket handling + sub-second
 * SQLite reads). The default timeout is therefore vastly larger than any
 * legitimate main-thread block yet vastly smaller than "forever". Opt out with
 * `CODEGRAPH_NO_WATCHDOG=1`; tune with `CODEGRAPH_WATCHDOG_TIMEOUT_MS`.
 */
import { Worker } from 'worker_threads';

/** Default: 60s — ~300× shorter than the 5h #850 wedge, far longer than any real main-thread block. */
export const DEFAULT_WATCHDOG_TIMEOUT_MS = 60_000;

export interface HeartbeatState {
  /** Last heartbeat counter the worker observed. */
  lastCounter: number;
  /** Consecutive checks the counter has NOT advanced. */
  staleChecks: number;
}

/**
 * Pure reducer for one worker check. `maxStaleChecks` consecutive no-progress
 * checks → wedged. Counting iterations (not wall-clock) is what makes this
 * robust to clock jumps / system sleep.
 */
export function stepHeartbeat(
  state: HeartbeatState,
  counter: number,
  maxStaleChecks: number
): { next: HeartbeatState; wedged: boolean } {
  if (counter !== state.lastCounter) {
    return { next: { lastCounter: counter, staleChecks: 0 }, wedged: false };
  }
  const staleChecks = state.staleChecks + 1;
  return {
    next: { lastCounter: counter, staleChecks },
    wedged: staleChecks >= maxStaleChecks,
  };
}

/** `true` for `1/true/yes/on` (case-insensitive); `false` otherwise. */
function isEnvTruthy(raw: string | undefined): boolean {
  if (!raw) return false;
  return ['1', 'true', 'yes', 'on'].includes(raw.trim().toLowerCase());
}

/** Parse the timeout env, falling back to the default for missing/invalid values. */
export function parseWatchdogTimeoutMs(
  raw: string | undefined,
  fallback: number = DEFAULT_WATCHDOG_TIMEOUT_MS
): number {
  if (raw === undefined) return fallback;
  const n = Number(raw);
  return Number.isFinite(n) && n > 0 ? n : fallback;
}

/** Derive a heartbeat/check cadence that fires several times inside the timeout window. */
export function deriveCheckIntervalMs(timeoutMs: number): number {
  return Math.min(2000, Math.max(50, Math.round(timeoutMs / 5)));
}

export interface WatchdogHandle {
  /** Stop heartbeating and terminate the worker. Idempotent. */
  stop(): void;
}

/**
 * The worker body, run via `new Worker(src, { eval: true })`. Inlined as a
 * string (not a shipped `.js`) so there is no dist-vs-src path to resolve — it
 * runs identically under `tsx` in tests and under the bundle in production.
 * Mirrors {@link stepHeartbeat}; keep the two in sync (the unit test pins the
 * algorithm, the integration test pins this exact body end-to-end).
 */
const WORKER_SOURCE = `
const { workerData } = require('worker_threads');
const fs = require('fs');
const beat = new Int32Array(workerData.sab);
const { checkMs, maxStaleChecks } = workerData;
let lastCounter = Atomics.load(beat, 0);
let staleChecks = 0;
const timer = setInterval(() => {
  const counter = Atomics.load(beat, 0);
  if (counter !== lastCounter) { lastCounter = counter; staleChecks = 0; return; }
  if (++staleChecks < maxStaleChecks) return;
  clearInterval(timer);
  const secs = Math.round((staleChecks * checkMs) / 1000);
  try {
    fs.writeSync(2, '[CodeGraph] Main thread unresponsive for ~' + secs + 's — killing the wedged process so a fresh one can start (#850). Disable with CODEGRAPH_NO_WATCHDOG=1.\\n');
  } catch (e) { /* stderr gone */ }
  try { process.kill(process.pid, 'SIGKILL'); } catch (e) { /* nothing left to try */ }
}, checkMs);
`;

/**
 * Install the main-thread liveness watchdog for a long-lived process. Returns a
 * handle to stop it, or `null` when disabled or when the worker can't be
 * spawned (degraded, never throws — a missing watchdog must never keep a
 * process from starting).
 */
export function installMainThreadWatchdog(): WatchdogHandle | null {
  if (isEnvTruthy(process.env.CODEGRAPH_NO_WATCHDOG)) return null;

  const timeoutMs = parseWatchdogTimeoutMs(process.env.CODEGRAPH_WATCHDOG_TIMEOUT_MS);
  const checkMs = deriveCheckIntervalMs(timeoutMs);
  const maxStaleChecks = Math.max(1, Math.ceil(timeoutMs / checkMs));

  // Single Int32 counter in shared memory. The main thread bumps it each tick;
  // the worker reads it. Atomics make the write visible across threads.
  const sab = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT);
  const beat = new Int32Array(sab);

  // The heartbeat: firing at all means the event loop is turning. unref'd so it
  // never keeps the process alive on its own (the server's socket does that).
  const heartbeat = setInterval(() => {
    Atomics.add(beat, 0, 1);
  }, checkMs);
  heartbeat.unref();

  let worker: Worker;
  try {
    worker = new Worker(WORKER_SOURCE, {
      eval: true,
      workerData: { sab, checkMs, maxStaleChecks },
    });
  } catch {
    // Worker threads unavailable — fall back to no watchdog rather than refuse
    // to start. Degraded (a future wedge wouldn't self-kill) but not broken.
    clearInterval(heartbeat);
    return null;
  }

  // A watchdog-worker error must never escalate to the global handler (which now
  // exits, #850): swallow it and run degraded.
  worker.on('error', () => { /* watchdog gone; nothing safe to do here */ });
  // Don't let the watchdog keep the process alive past its real work.
  worker.unref();

  let stopped = false;
  return {
    stop(): void {
      if (stopped) return;
      stopped = true;
      clearInterval(heartbeat);
      void worker.terminate();
    },
  };
}