feat(resolution): memory-aware, cgroup-honest worker-pool sizing + CODEGRAPH_RESOLVE_WORKERS (#1333)

Pool sizing used os.cpus().length, which enumerates the HOST's CPUs: inside
a 2-CPU cpuset it sized 6 resolver workers (the §7a.1 false-'sequential'
premise) and 8 parse workers, and at true 8-core concurrency six ~1GB
workers OOM-killed a 7GB container (oom_kill=5) mid-synthesis — sizing had
no memory term and no override knob.

resolvePoolSize (pure, matrix-tested): explicit CODEGRAPH_RESOLVE_WORKERS
override (0 disables, cap 16); CPU term max(2, min(availableParallelism-1,
6)) — cpuset-honest, floored at 2 so true 2-core boxes keep pooled
synthesis's ~2×; memory term floor(budget*0.7 / clamp(0.2*dbSize, 256MB,
1.5GB)) with budget = min(freemem, cgroup v2/v1 headroom). Parse pool's
core input switches to availableParallelism. Dev machines are unchanged
(still 6 workers); the 8c/7GB kernel-scale container now sizes 4.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

src/extraction/index.ts

@@ -1615,8 +1615,12 @@ export class ExtractionOrchestrator {
     let pool: ParseWorkerPool | null = null;
     if (useWorker) {
       // CODEGRAPH_PARSE_WORKERS: explicit worker count; 1 = the old single-worker
-      // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8).
-      const poolSize = resolveParsePoolSize(process.env.CODEGRAPH_PARSE_WORKERS, os.cpus().length);
+      // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8),
+      // with cores from availableParallelism — cpuset/affinity-honest, where
+      // os.cpus() enumerates the host's CPUs and spawned 8 wasm workers (and
+      // their grammar heaps) inside a 2-CPU container for zero extra
+      // throughput (§7a.1).
+      const poolSize = resolveParsePoolSize(process.env.CODEGRAPH_PARSE_WORKERS, os.availableParallelism());
       // Read each needed grammar's WASM ONCE here and hand the bytes to every
       // worker, so spawns/respawns load grammars from memory instead of
       // re-reading them from disk (#1231: on an HDD, respawn re-reads amplify

src/resolution/memory-budget.ts

@@ -0,0 +1,59 @@
+/**
+ * Memory headroom for worker-pool sizing — cgroup-honest on Linux.
+ *
+ * `os.freemem()` reads /proc/meminfo, which inside a container reports the
+ * HOST's (or VM's) memory, not the cgroup's — the same blindness os.cpus()
+ * has for cpusets. A resolver pool sized by cores alone OOM-killed a
+ * kernel-scale index in a 7GB-capped container (migration plan §7a.1:
+ * oom_kill=5, six ~1GB workers at true 8-core concurrency), so pool sizing
+ * combines a CPU term with the memory headroom this module reports.
+ */
+
+import * as fs from 'fs';
+import * as os from 'os';
+
+/** Parse a cgroup value file: numeric bytes, or null for absent/'max'. */
+function readCgroupBytes(path: string): number | null {
+  try {
+    const raw = fs.readFileSync(path, 'utf8').trim();
+    if (raw === 'max') return null;
+    const n = Number.parseInt(raw, 10);
+    return Number.isFinite(n) && n >= 0 ? n : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Available headroom under the cgroup memory limit (v2 then v1), or null
+ * when uncontained (no limit, non-Linux, or unreadable). Never throws.
+ */
+export function cgroupMemoryAvailable(): number | null {
+  if (process.platform !== 'linux') return null;
+  // v2 unified hierarchy
+  const v2Max = readCgroupBytes('/sys/fs/cgroup/memory.max');
+  if (v2Max !== null) {
+    const current = readCgroupBytes('/sys/fs/cgroup/memory.current') ?? 0;
+    return Math.max(0, v2Max - current);
+  }
+  // v1
+  const v1Limit = readCgroupBytes('/sys/fs/cgroup/memory/memory.limit_in_bytes');
+  // v1 reports "no limit" as a huge sentinel (~PAGE_COUNTER_MAX); treat
+  // anything at or beyond half the address-space-ish range as uncontained.
+  if (v1Limit !== null && v1Limit < 2 ** 60) {
+    const usage = readCgroupBytes('/sys/fs/cgroup/memory/memory.usage_in_bytes') ?? 0;
+    return Math.max(0, v1Limit - usage);
+  }
+  return null;
+}
+
+/**
+ * The budget pool sizing divides: the smaller of system free memory and the
+ * cgroup headroom (when contained). Conservative by construction — both
+ * numbers shrink as the process itself grows.
+ */
+export function memoryBudgetBytes(): number {
+  const free = os.freemem();
+  const cgroup = cgroupMemoryAvailable();
+  return cgroup === null ? free : Math.min(free, cgroup);
+}

src/resolution/resolver-pool.ts

@@ -15,6 +15,7 @@ import * as path from 'path';
 import * as os from 'os';
 import type { Edge, UnresolvedReference } from '../types';
 import type { ResolvedRef, UnresolvedRef } from './types';
+import { memoryBudgetBytes } from './memory-budget';
 
 /** One synthesis pass's output: its edge list + worker-measured wall clock. */
 export interface SynthPassResult {
@@ -64,17 +65,70 @@ export class ResolverPool {
   private synthWaiters = new Map<number, { resolve: (r: SynthPassResult) => void; reject: (e: Error) => void }>();
   private failed: Error | null = null;
 
+  /**
+   * Pool size from CPU headroom, memory headroom, and the explicit override.
+   * Pure — every input injected — so the whole matrix is unit-testable.
+   *
+   * CPU term: `availableParallelism` (cpuset/affinity-honest — `os.cpus()`
+   * enumerates the host's CPUs and sized SIX workers inside a 2-CPU cpuset,
+   * §7a.1's false-premise finding), minus one for the persisting main thread,
+   * floored at 2 so a true 2-core box keeps the pool's ~2× on synthesis,
+   * capped at the long-standing 6.
+   *
+   * Memory term: workers hold real heap at scale (~1GB each against a 4.6GB
+   * kernel-scale DB — six of them OOM-killed a 7GB container once real
+   * 8-core concurrency let them peak simultaneously). Estimate per-worker
+   * cost from the DB size, keep 30% of the budget for the main thread, and
+   * let the smaller term win. Below 2 workers the pool isn't worth its boot
+   * cost — callers get null and stay sequential.
+   */
+  static resolvePoolSize(opts: {
+    explicit?: string;
+    availableParallelism: number;
+    memoryBudget: number;
+    dbSizeBytes: number;
+  }): number | null {
+    if (opts.explicit !== undefined && opts.explicit !== '') {
+      const n = Number.parseInt(opts.explicit, 10);
+      if (Number.isFinite(n)) {
+        if (n <= 0) return null;
+        return Math.min(n, 16);
+      }
+    }
+    const cpuCap = Math.max(2, Math.min(opts.availableParallelism - 1, 6));
+    const perWorker = Math.min(Math.max(opts.dbSizeBytes * 0.2, 256 * 1024 * 1024), 1.5 * 1024 * 1024 * 1024);
+    const memCap = Math.floor((opts.memoryBudget * 0.7) / perWorker);
+    const size = Math.min(cpuCap, memCap);
+    return size >= 2 ? size : null;
+  }
+
   /**
    * Create a pool when the compiled worker exists (absent when running from
    * source in tests → callers use the sequential path), the kill switch is
-   * off, and the machine has cores to spare. Returns null otherwise.
+   * off, and the machine has the cores AND memory to carry it. Returns null
+   * otherwise. `CODEGRAPH_RESOLVE_WORKERS` overrides the computed size
+   * (0 disables the pool; values are capped at 16).
    */
   static tryCreate(dbPath: string, projectRoot: string): ResolverPool | null {
     if (process.env.CODEGRAPH_NO_PARALLEL_RESOLVE === '1') return null;
     const workerScript = path.join(__dirname, 'resolver-worker.js');
     if (!fs.existsSync(workerScript)) return null;
-    const size = Math.max(1, Math.min(os.cpus().length - 2, 6));
-    if (size < 2) return null;
+    let dbSizeBytes = 0;
+    try {
+      dbSizeBytes = fs.statSync(dbPath).size;
+    } catch { /* fresh/missing file — the 256MB per-worker floor applies */ }
+    const size = ResolverPool.resolvePoolSize({
+      explicit: process.env.CODEGRAPH_RESOLVE_WORKERS,
+      availableParallelism: os.availableParallelism(),
+      memoryBudget: memoryBudgetBytes(),
+      dbSizeBytes,
+    });
+    if (size === null) return null;
+    if (process.env.CODEGRAPH_SYNTH_TIMINGS) {
+      console.error(
+        `[pool-timing] pool size=${size} (ap=${os.availableParallelism()} budget=${Math.round(memoryBudgetBytes() / 1024 / 1024)}MB db=${Math.round(dbSizeBytes / 1024 / 1024)}MB)`
+      );
+    }
     try {
       return new ResolverPool(workerScript, dbPath, projectRoot, size);
     } catch {