fix(watcher): bound fd/watch cost with a native fs.watch hybrid (#644, #496, #555, #628, #579) (#650)
chokidar v4 holds one OS file descriptor per watched file on macOS (libuv's kqueue backend registers an fd per vnode; fsevents is installed but v4 no longer uses it). On a large project the `serve --mcp` daemon accumulated tens of thousands of open REG descriptors and exhausted kern.maxfiles — crashing unrelated processes system-wide with ENFILE. #276 only trimmed the count by ignoring directories; the source tree still cost one fd per file. Replace chokidar with a pure-JS native fs.watch hybrid, keeping codegraph's zero-native-addon "any OS builds any bundle" invariant: - macOS / Windows: a single recursive fs.watch (one FSEvents stream / ReadDirectoryChangesW handle) -> O(1) descriptors regardless of repo size. - Linux: one inotify watch per directory (O(dirs), dynamic add for new dirs, capped via CODEGRAPH_MAX_DIR_WATCHES) instead of per-file watches. Validated empirically: macOS 0 extra fds at 6k and 12k files; Linux 31 inotify watches at 6k files (per-file would be 6k); Windows recursive catches nested and new-directory edits. Full test suite green. Tests drive the watcher through an inertForTests seam (no OS watcher) for determinism under parallel vitest, with one real-fs end-to-end test exercising the genuine native path. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
434be1da58
commit
c9559d9991
@@ -11,27 +11,22 @@
|
||||
* decides whether to Read the specific stale file. These tests exercise
|
||||
* the full real path: real CodeGraph index + real ToolHandler.execute().
|
||||
*
|
||||
* **chokidar is mocked** (see __helpers__/chokidar-mock.ts): the real
|
||||
* FSEvents/inotify event delivery is non-deterministic under parallel
|
||||
* vitest execution and produced a consistent ~30% failure rate on these
|
||||
* tests when run inside the full suite. The mock replaces chokidar with
|
||||
* a controllable EventEmitter so the tests synthesize file events
|
||||
* deterministically via `triggerFileEvent(...)` instead of waiting on
|
||||
* the OS-level watcher to deliver. The watcher's actual debounce timer
|
||||
* (real setTimeout) is left untouched.
|
||||
* **Event delivery uses a synthetic seam** (`__emitWatchEventForTests`): the
|
||||
* real native fs.watch (FSEvents/inotify) delivery is non-deterministic under
|
||||
* parallel vitest execution and produced a consistent ~30% failure rate on
|
||||
* these tests when run inside the full suite. The seam drives the watcher's
|
||||
* pending-set pipeline directly so the tests synthesize file events
|
||||
* deterministically. The watcher's actual debounce timer (real setTimeout) is
|
||||
* left untouched.
|
||||
*/
|
||||
|
||||
import { vi } from 'vitest';
|
||||
// Hoisted: chokidar is replaced by the controllable mock for this file.
|
||||
vi.mock('chokidar', async () => (await import('./__helpers__/chokidar-mock')).chokidarMockModule);
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import CodeGraph from '../src/index';
|
||||
import { ToolHandler } from '../src/mcp/tools';
|
||||
import { triggerFileEvent } from './__helpers__/chokidar-mock';
|
||||
import { __emitWatchEventForTests } from '../src/sync/watcher';
|
||||
|
||||
function waitFor(condition: () => boolean, timeoutMs = 2000, intervalMs = 25): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -83,7 +78,7 @@ describe('MCP staleness banner', () => {
|
||||
|
||||
it('prepends a stale banner when the response references a pending file', async () => {
|
||||
// Long debounce so the edit lingers in pendingFiles while we query.
|
||||
cg.watch({ debounceMs: 4000 });
|
||||
cg.watch({ debounceMs: 4000, inertForTests: true });
|
||||
await cg.waitUntilWatcherReady();
|
||||
|
||||
// Real disk write so a later sync (if it fires) sees the new content,
|
||||
@@ -93,7 +88,7 @@ describe('MCP staleness banner', () => {
|
||||
path.join(testDir, 'src', 'alpha-only.ts'),
|
||||
'export function alphaOnly() { return 99; }\n',
|
||||
);
|
||||
triggerFileEvent(testDir, 'change', 'src/alpha-only.ts');
|
||||
__emitWatchEventForTests(testDir, 'src/alpha-only.ts');
|
||||
|
||||
// With mocked chokidar this is synchronous — keep the wait just to
|
||||
// exercise the realistic shape (the watcher's `chokidarReady` gate
|
||||
@@ -114,7 +109,7 @@ describe('MCP staleness banner', () => {
|
||||
});
|
||||
|
||||
it('uses the footer (not the banner) when pending files are not referenced', async () => {
|
||||
cg.watch({ debounceMs: 4000 });
|
||||
cg.watch({ debounceMs: 4000, inertForTests: true });
|
||||
await cg.waitUntilWatcherReady();
|
||||
|
||||
// Edit bravo-only.ts but search for the alphaOnly symbol, whose hit is
|
||||
@@ -124,7 +119,7 @@ describe('MCP staleness banner', () => {
|
||||
path.join(testDir, 'src', 'bravo-only.ts'),
|
||||
'export function bravoOnly() { return 22; }\n',
|
||||
);
|
||||
triggerFileEvent(testDir, 'change', 'src/bravo-only.ts');
|
||||
__emitWatchEventForTests(testDir, 'src/bravo-only.ts');
|
||||
await waitFor(() => cg.getPendingFiles().some((p) => p.path === 'src/bravo-only.ts'));
|
||||
|
||||
const res = await handler.execute('codegraph_search', { query: 'alphaOnly' });
|
||||
@@ -136,14 +131,14 @@ describe('MCP staleness banner', () => {
|
||||
});
|
||||
|
||||
it('drops the banner once the sync completes and clears the pending entry', async () => {
|
||||
cg.watch({ debounceMs: 200 });
|
||||
cg.watch({ debounceMs: 200, inertForTests: true });
|
||||
await cg.waitUntilWatcherReady();
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(testDir, 'src', 'alpha-only.ts'),
|
||||
'export function alphaOnly() { return 7; }\n',
|
||||
);
|
||||
triggerFileEvent(testDir, 'change', 'src/alpha-only.ts');
|
||||
__emitWatchEventForTests(testDir, 'src/alpha-only.ts');
|
||||
// Wait through debounce (200ms) + sync; pendingFiles drains back to empty.
|
||||
await waitFor(() => cg.getPendingFiles().length === 0, 3000);
|
||||
|
||||
@@ -154,14 +149,14 @@ describe('MCP staleness banner', () => {
|
||||
});
|
||||
|
||||
it('lists pending files under "Pending sync" in codegraph_status', async () => {
|
||||
cg.watch({ debounceMs: 4000 });
|
||||
cg.watch({ debounceMs: 4000, inertForTests: true });
|
||||
await cg.waitUntilWatcherReady();
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(testDir, 'src', 'charlie-only.ts'),
|
||||
'export function charlieOnly() { return 33; }\n',
|
||||
);
|
||||
triggerFileEvent(testDir, 'change', 'src/charlie-only.ts');
|
||||
__emitWatchEventForTests(testDir, 'src/charlie-only.ts');
|
||||
await waitFor(() => cg.getPendingFiles().some((p) => p.path === 'src/charlie-only.ts'));
|
||||
|
||||
const res = await handler.execute('codegraph_status', {});
|
||||
|
||||
Reference in New Issue
Block a user