fix(watcher): exclude ignored dirs before watching to prevent inotify exhaustion (#276)
The file watcher registered a recursive watch over the entire project (node_modules, build output, caches included) and filtered only in the callback — exhausting the Linux inotify budget on large repos (#276). It now uses chokidar and excludes the same directories the indexer ignores (built-in default-ignore set + the project .gitignore) BEFORE registering a watch, so the watch count on a 900-dir node_modules drops from ~1200 to ~14 even with no .gitignore. Stacks with the shared daemon (#411): one watcher across agents, now small. Also hardens the #411 daemon lockfile against a concurrent-startup race the new watcher timing made reproducible — the lock is now created atomically with its content (temp-write + hard-link), so racing daemons can never both win. Validated on macOS, Linux (Docker), and Windows (chokidar + fs.linkSync on NTFS). Co-Authored-By: Colby McHenry <me@colbymchenry.com> Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Lorenzo Feng
co-authored by
Colby McHenry
Claude Opus 4.7
Colby McHenry
Claude Opus 4.7
parent
995da54430
commit
b09b23cf54
+80
-42
@@ -1,15 +1,24 @@
|
||||
/**
|
||||
* File Watcher
|
||||
*
|
||||
* Watches the project directory for file changes and triggers
|
||||
* debounced sync operations to keep the code graph up-to-date.
|
||||
* Watches the project directory for file changes and triggers debounced sync
|
||||
* operations to keep the code graph up-to-date.
|
||||
*
|
||||
* Uses Node.js native fs.watch with recursive mode (macOS FSEvents,
|
||||
* Windows ReadDirectoryChangesW, Linux inotify on Node 19+).
|
||||
* Uses chokidar, whose `ignored` callback filters directories BEFORE they are
|
||||
* watched — so we never register inotify watches on excluded trees like
|
||||
* node_modules/, dist/, .git/ (fixes #276: recursive fs.watch exhausted the
|
||||
* kernel watch budget on large repos). The ignore decision reuses the indexer's
|
||||
* `buildDefaultIgnore` (built-in default-ignore dirs + the project's .gitignore)
|
||||
* so the watcher watches exactly the set the indexer indexes — in particular,
|
||||
* node_modules/build/cache dirs are excluded even when the repo has no
|
||||
* .gitignore (#407), which a .gitignore-only filter would miss.
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import { isSourceFile } from '../extraction';
|
||||
import * as path from 'path';
|
||||
import type { Stats } from 'fs';
|
||||
import chokidar, { FSWatcher } from 'chokidar';
|
||||
import type { Ignore } from 'ignore';
|
||||
import { isSourceFile, buildDefaultIgnore } from '../extraction';
|
||||
import { logDebug, logWarn } from '../errors';
|
||||
import { normalizePath } from '../utils';
|
||||
import { watchDisabledReason } from './watch-policy';
|
||||
@@ -41,17 +50,22 @@ export interface WatchOptions {
|
||||
* debounced sync operations via a provided callback.
|
||||
*
|
||||
* Design goals:
|
||||
* - Minimal resource usage (native OS file events, no polling)
|
||||
* - Minimal resource usage (chokidar filters excluded directories before
|
||||
* registering an inotify watch — see module docs / #276)
|
||||
* - Debounced to avoid thrashing on rapid saves
|
||||
* - Filters to supported source files by extension
|
||||
* - Ignores .codegraph/ directory changes
|
||||
* - Ignores .codegraph/ and .git/ regardless of .gitignore
|
||||
*/
|
||||
export class FileWatcher {
|
||||
private watcher: fs.FSWatcher | null = null;
|
||||
private watcher: FSWatcher | null = null;
|
||||
private debounceTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private hasChanges = false;
|
||||
private syncing = false;
|
||||
private stopped = false;
|
||||
// The shared ignore matcher (built-in defaults + project .gitignore), built
|
||||
// once at start(). Same source of truth the indexer uses, so watcher scope
|
||||
// can never diverge from index scope.
|
||||
private ignoreMatcher: Ignore | null = null;
|
||||
|
||||
private readonly projectRoot: string;
|
||||
private readonly debounceMs: number;
|
||||
@@ -79,61 +93,84 @@ export class FileWatcher {
|
||||
if (this.watcher) return true; // Already watching
|
||||
this.stopped = false;
|
||||
|
||||
// Some environments make recursive fs.watch unusable — most notably WSL2
|
||||
// /mnt/ drives, where setup blocks long enough to break MCP startup
|
||||
// handshakes (issue #199). Skip watching there; callers fall back to
|
||||
// manual `codegraph sync` or the git sync hooks.
|
||||
// Some environments make filesystem watching unusable — most notably
|
||||
// WSL2 /mnt/ drives, where the underlying fs.watch calls block long
|
||||
// enough to break MCP startup handshakes (issue #199). Skip watching
|
||||
// there; callers fall back to manual `codegraph sync` or git sync hooks.
|
||||
const disabledReason = watchDisabledReason(this.projectRoot);
|
||||
if (disabledReason) {
|
||||
logDebug('File watcher disabled', { reason: disabledReason, projectRoot: this.projectRoot });
|
||||
return false;
|
||||
}
|
||||
|
||||
// Reuse the indexer's ignore set so the watcher and indexer agree on scope.
|
||||
// chokidar only registers an inotify watch on directories that pass this
|
||||
// filter — that's the #276 fix.
|
||||
this.ignoreMatcher = buildDefaultIgnore(this.projectRoot);
|
||||
|
||||
try {
|
||||
this.watcher = fs.watch(
|
||||
this.projectRoot,
|
||||
{ recursive: true },
|
||||
(_eventType, filename) => {
|
||||
if (!filename || this.stopped) return;
|
||||
this.watcher = chokidar.watch(this.projectRoot, {
|
||||
// chokidar calls this for every path it encounters and only watches
|
||||
// those that pass — so excluded trees (node_modules/, dist/, .git/, …)
|
||||
// never get an inotify watch in the first place.
|
||||
ignored: (testPath: string, stats?: Stats) => this.shouldIgnore(testPath, stats),
|
||||
});
|
||||
|
||||
// Normalize path separators
|
||||
const normalized = normalizePath(filename);
|
||||
// chokidar emits 'all' for every event type; we only sync source files.
|
||||
this.watcher.on('all', (_event: string, filePath: string) => {
|
||||
if (this.stopped) return;
|
||||
|
||||
// Ignore .codegraph/ directory changes (our own DB writes)
|
||||
if (
|
||||
normalized === '.codegraph' ||
|
||||
normalized.startsWith('.codegraph/') ||
|
||||
normalized.startsWith('.codegraph\\')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const normalized = normalizePath(path.relative(this.projectRoot, filePath));
|
||||
|
||||
// Only sync changes to files we can actually parse.
|
||||
if (!isSourceFile(normalized)) {
|
||||
return;
|
||||
}
|
||||
// Defense in depth: `ignored` should already keep these out, but events
|
||||
// can still arrive during setup or via symlink traversal.
|
||||
if (this.isAlwaysIgnored(normalized)) return;
|
||||
if (!isSourceFile(normalized)) return;
|
||||
|
||||
logDebug('File change detected', { file: normalized });
|
||||
this.hasChanges = true;
|
||||
this.scheduleSync();
|
||||
}
|
||||
);
|
||||
logDebug('File change detected', { file: normalized });
|
||||
this.hasChanges = true;
|
||||
this.scheduleSync();
|
||||
});
|
||||
|
||||
// Handle watcher errors gracefully
|
||||
this.watcher.on('error', (err) => {
|
||||
// Handle watcher errors gracefully — don't crash, the user can restart.
|
||||
this.watcher.on('error', (err: unknown) => {
|
||||
logWarn('File watcher error', { error: String(err) });
|
||||
// Don't crash — watcher may recover or user can restart
|
||||
});
|
||||
|
||||
logDebug('File watcher started', { projectRoot: this.projectRoot, debounceMs: this.debounceMs });
|
||||
return true;
|
||||
} catch (err) {
|
||||
// Recursive watch not supported (e.g., Linux < Node 19)
|
||||
logWarn('Could not start file watcher — recursive fs.watch not supported on this platform', { error: String(err) });
|
||||
// Watcher setup failed (e.g., permission denied, missing directory).
|
||||
logWarn('Could not start file watcher', { error: String(err) });
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Our own dirs are always ignored, regardless of .gitignore. */
|
||||
private isAlwaysIgnored(rel: string): boolean {
|
||||
return (
|
||||
rel === '.codegraph' || rel.startsWith('.codegraph/') ||
|
||||
rel === '.git' || rel.startsWith('.git/')
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* chokidar `ignored` predicate — true for any path that should NOT be watched.
|
||||
* Uses chokidar's provided `stats` to decide directory-vs-file so a dir-only
|
||||
* rule like `build/` matches, without an extra `statSync` per path.
|
||||
*/
|
||||
private shouldIgnore(testPath: string, stats?: Stats): boolean {
|
||||
const rel = normalizePath(path.relative(this.projectRoot, testPath));
|
||||
if (!rel || rel === '.' || rel.startsWith('..')) return false; // root / outside
|
||||
if (this.isAlwaysIgnored(rel)) return true;
|
||||
if (!this.ignoreMatcher) return false;
|
||||
if (stats) {
|
||||
return this.ignoreMatcher.ignores(stats.isDirectory() ? rel + '/' : rel);
|
||||
}
|
||||
// Stats unknown: test both forms so a directory match isn't missed.
|
||||
return this.ignoreMatcher.ignores(rel) || this.ignoreMatcher.ignores(rel + '/');
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop watching for file changes.
|
||||
*/
|
||||
@@ -151,6 +188,7 @@ export class FileWatcher {
|
||||
}
|
||||
|
||||
this.hasChanges = false;
|
||||
this.ignoreMatcher = null;
|
||||
logDebug('File watcher stopped');
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user