diff --git a/CHANGELOG.md b/CHANGELOG.md index 288f80b..5910279 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). #### MCP / indexing +- **Watcher scope now matches `git ls-files --exclude-standard` (#1728).** `buildDefaultIgnore` / `buildScopeIgnore` read `.git/info/exclude` and `core.excludesFile` (not only the root `.gitignore`), and seed directories git reports as ignored-untracked so nested `.gitignore` effects prune the live watcher the same way the indexer skips them. Single-file auto-sync was already incremental (`pendingFiles` → scoped `sync({ paths })`); the remaining gap was watching trees git had excluded. + - **Live sync no longer lets the write-ahead log grow without a bound when a reader is holding it open (#1539).** Incremental sync now uses the same writer pause that full indexing already used, and if checkpointing still cannot finish once the log is past its documented size limit — typically because the query pool is reading at the same time — sync stops with a clear error instead of keeping writing until the disk fills. The previous behaviour could leave a multi-tens-of-gigabyte log beside a few-gigabyte index on a large project. Close concurrent readers and retry, or raise `CODEGRAPH_WAL_VALVE_MB` if the limit is too tight for the project. - **A second `codegraph serve --mcp` on the same project no longer silently kills auto-sync (#1740).** Direct mode (`CODEGRAPH_NO_DAEMON=1` or proxy→in-process fallback) now takes an exclusive `.codegraph/writer.pid` lock; a second writer exits immediately with guidance to stop the other server or unset `CODEGRAPH_NO_DAEMON` so clients share the daemon. The shared daemon already multiplexes N clients onto one watcher — this closes the same-OS dual-direct gap the docs warned about for Windows/WSL but did not guard. diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index 60cbd10..5478bda 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -7793,6 +7793,82 @@ describe('Nested non-submodule git repos', () => { expect(ig.ignores('dist/')).toBe(true); // valid rule survives expect(ig.ignores('src/app.ts')).toBe(false); }); + + it('buildDefaultIgnore honors .git/info/exclude (#1728)', async () => { + const { execFileSync } = await import('child_process'); + const git = (cwd: string, ...args: string[]) => + execFileSync('git', args, { cwd, stdio: 'pipe' }); + + const root = path.join(tempDir, 'exclude-root'); + fs.mkdirSync(root, { recursive: true }); + git(root, 'init', '-q'); + fs.writeFileSync(path.join(root, 'src.ts'), 'export const x = 1;\n'); + fs.mkdirSync(path.join(root, '.claude', 'worktrees', 'agent-1'), { recursive: true }); + fs.writeFileSync( + path.join(root, '.claude', 'worktrees', 'agent-1', 'src.ts'), + 'export const w = 1;\n', + ); + // Not in .gitignore — only in info/exclude (the reporter's exact shape). + fs.writeFileSync( + path.join(root, '.git', 'info', 'exclude'), + '**/.claude/worktrees/\n', + ); + + const ig = buildDefaultIgnore(root); + expect(ig.ignores('src.ts')).toBe(false); + expect(ig.ignores('.claude/worktrees/agent-1/src.ts')).toBe(true); + expect(ig.ignores('.claude/worktrees/')).toBe(true); + + // ScopeIgnore (watcher path) agrees, including via git ignored-dir seeding. + const scope = buildScopeIgnore(root); + expect(scope.ignores('src.ts')).toBe(false); + expect(scope.ignores('.claude/worktrees/agent-1/')).toBe(true); + expect(scope.ignores('.claude/worktrees/agent-1/src.ts')).toBe(true); + }); + + it('buildDefaultIgnore honors core.excludesFile (#1728)', async () => { + const { execFileSync } = await import('child_process'); + const git = (cwd: string, ...args: string[]) => + execFileSync('git', args, { cwd, stdio: 'pipe' }); + + const root = path.join(tempDir, 'excludesfile-root'); + fs.mkdirSync(root, { recursive: true }); + git(root, 'init', '-q'); + const globalExcludes = path.join(tempDir, 'global-excludes'); + fs.writeFileSync(globalExcludes, 'scratch/\n'); + git(root, 'config', 'core.excludesFile', globalExcludes); + fs.mkdirSync(path.join(root, 'scratch'), { recursive: true }); + fs.writeFileSync(path.join(root, 'scratch', 'tmp.ts'), 'export const t = 1;\n'); + fs.writeFileSync(path.join(root, 'app.ts'), 'export const a = 1;\n'); + + const ig = buildDefaultIgnore(root); + expect(ig.ignores('app.ts')).toBe(false); + expect(ig.ignores('scratch/')).toBe(true); + expect(ig.ignores('scratch/tmp.ts')).toBe(true); + }); + + it('buildScopeIgnore prunes dirs ignored only by a nested .gitignore (#1728)', async () => { + const { execFileSync } = await import('child_process'); + const git = (cwd: string, ...args: string[]) => + execFileSync('git', args, { cwd, stdio: 'pipe' }); + + const root = path.join(tempDir, 'nested-gi-root'); + fs.mkdirSync(path.join(root, 'pkg', 'build'), { recursive: true }); + git(root, 'init', '-q'); + git(root, 'config', 'user.email', 'test@test.com'); + git(root, 'config', 'user.name', 'Test'); + fs.writeFileSync(path.join(root, 'pkg', 'app.ts'), 'export const a = 1;\n'); + fs.writeFileSync(path.join(root, 'pkg', 'build', 'out.ts'), 'export const o = 1;\n'); + fs.writeFileSync(path.join(root, 'pkg', '.gitignore'), 'build/\n'); + // Commit only the non-ignored file so git still reports build/ as ignored-other. + git(root, 'add', 'pkg/app.ts', 'pkg/.gitignore'); + git(root, 'commit', '-q', '-m', 'init'); + + const scope = buildScopeIgnore(root); + expect(scope.ignores('pkg/app.ts')).toBe(false); + expect(scope.ignores('pkg/build/')).toBe(true); + expect(scope.ignores('pkg/build/out.ts')).toBe(true); + }); }); // ============================================================================= diff --git a/__tests__/watcher.test.ts b/__tests__/watcher.test.ts index 942fd5b..6b87a55 100644 --- a/__tests__/watcher.test.ts +++ b/__tests__/watcher.test.ts @@ -608,6 +608,42 @@ describe('FileWatcher', () => { watcher.stop(); }); + it('info/exclude patterns drop worktree paths from pending (#1728)', async () => { + const { execFileSync } = await import('child_process'); + // Re-init the testDir as a real git repo so buildScopeIgnore can read + // .git/info/exclude (createTempDir fixtures are usually plain dirs). + execFileSync('git', ['init', '-q'], { cwd: testDir, stdio: 'pipe' }); + fs.mkdirSync(path.join(testDir, '.claude', 'worktrees', 'w1', 'src'), { recursive: true }); + fs.writeFileSync( + path.join(testDir, '.claude', 'worktrees', 'w1', 'src', 'x.ts'), + 'export const x = 1;\n', + ); + fs.writeFileSync( + path.join(testDir, '.git', 'info', 'exclude'), + '**/.claude/worktrees/\n', + ); + + const syncFn = vi.fn().mockResolvedValue({ filesChanged: 0, durationMs: 0 }); + const watcher = newWatcher(syncFn, { debounceMs: 100 }); + watcher.start(); + await watcher.waitUntilReady(); + + __emitWatchEventForTests(testDir, '.claude/worktrees/w1/src/x.ts'); + expect(watcher.getPendingFiles().map((p) => p.path)).not.toContain( + '.claude/worktrees/w1/src/x.ts', + ); + await new Promise((r) => setTimeout(r, 300)); + expect(syncFn).not.toHaveBeenCalled(); + + // In-scope edits still schedule a scoped sync. + fs.writeFileSync(path.join(testDir, 'src', 'ok.ts'), 'export const ok = 1;\n'); + __emitWatchEventForTests(testDir, 'src/ok.ts'); + await waitFor(() => syncFn.mock.calls.length > 0); + expect(syncFn.mock.calls[0]![0]).toEqual(['src/ok.ts']); + + watcher.stop(); + }); + it('a nested .gitignore inside the scope forces a full sync', async () => { const syncFn = vi.fn().mockResolvedValue({ filesChanged: 0, durationMs: 0 }); const watcher = newWatcher(syncFn, { debounceMs: 100 }); diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 39c6aa8..c2bb2f3 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -301,17 +301,117 @@ function readGitignorePatterns(giPath: string): string { return kept.join('\n'); } +/** + * Resolve the repository GIT_DIR for `repoRoot` (a `.git` directory, or the + * target of a `.git` file pointer). Null when this isn't a git checkout. + */ +function resolveGitDir(repoRoot: string): string | null { + const gitPath = path.join(repoRoot, '.git'); + let st: fs.Stats; + try { + st = fs.statSync(gitPath); + } catch { + return null; + } + if (st.isDirectory()) return gitPath; + if (!st.isFile()) return null; + try { + const raw = fs.readFileSync(gitPath, 'utf8').match(/^gitdir:\s*(.+)$/m)?.[1]?.trim(); + if (!raw) return null; + return path.isAbsolute(raw) ? path.normalize(raw) : path.resolve(repoRoot, raw); + } catch { + return null; + } +} + +/** Expand a leading `~/` the way git does for `core.excludesFile`. */ +function expandUserPath(p: string): string { + if (p === '~') return os.homedir(); + if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2)); + return p; +} + +/** + * Root-relative exclude patterns from git sources that are NOT the root + * `.gitignore`: `.git/info/exclude` and `core.excludesFile`. Same semantics as + * the root `.gitignore`, so they merge into {@link buildDefaultIgnore}. Without + * these, the watcher / FS-walk scope silently diverged from + * `git ls-files --exclude-standard` (#1728). + */ +function readGitExcludeExtraPatterns(rootDir: string): string { + const chunks: string[] = []; + const gitDir = resolveGitDir(rootDir); + if (gitDir) { + const excludePath = path.join(gitDir, 'info', 'exclude'); + if (fs.existsSync(excludePath)) { + const patterns = readGitignorePatterns(excludePath); + if (patterns) chunks.push(patterns); + } + } + try { + const configured = execFileSync( + 'git', + ['-C', rootDir, 'config', '--get', 'core.excludesFile'], + { encoding: 'utf8', timeout: 5_000, stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim(); + if (configured) { + const abs = expandUserPath(configured); + if (fs.existsSync(abs)) { + const patterns = readGitignorePatterns(abs); + if (patterns) chunks.push(patterns); + } + } + } catch { + // No git, unset, or timeout — leave extras empty. + } + return chunks.join('\n'); +} + +/** + * Directories `git ls-files -o -i --exclude-standard --directory` reports as + * ignored-untracked. Seeded into {@link ScopeIgnore} so nested `.gitignore` + * effects (and any exclude-standard rule the flat matcher might miss) prune the + * watcher the same way the indexer skips them (#1728). + */ +function listGitIgnoredDirectories(rootDir: string): string[] { + try { + const out = execFileSync( + 'git', + ['-C', rootDir, 'ls-files', '-z', '-o', '-i', '--exclude-standard', '--directory'], + { + encoding: 'utf8', + timeout: 60_000, + maxBuffer: 50 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'ignore'], + }, + ); + const dirs: string[] = []; + for (const entry of out.split('\0')) { + if (!entry) continue; + dirs.push(entry.endsWith('/') ? entry : `${entry}/`); + } + return dirs; + } catch { + return []; + } +} + /** * An `ignore` matcher seeded with the built-in defaults, merged with the project's - * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default. Shared - * by both enumeration paths so behavior is identical with or without git — and so - * the defaults apply to tracked files too (committing a dependency dir doesn't make - * it project code; the explicit `.gitignore` negation is the only opt-in). + * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default, plus + * git's other root-relative exclude files (`.git/info/exclude`, `core.excludesFile`) + * so watcher / FS-walk scope matches `git ls-files --exclude-standard` (#1728). + * Shared by both enumeration paths so behavior is identical with or without git — + * and so the defaults apply to tracked files too (committing a dependency dir + * doesn't make it project code; the explicit `.gitignore` negation is the only + * opt-in). */ export function buildDefaultIgnore(rootDir: string): Ignore { const ig = ignore().add(DEFAULT_IGNORE_PATTERNS); const rootGitignore = path.join(rootDir, '.gitignore'); if (fs.existsSync(rootGitignore)) ig.add(readGitignorePatterns(rootGitignore)); + const extra = readGitExcludeExtraPatterns(rootDir); + if (extra) ig.add(extra); return ig; } @@ -630,14 +730,17 @@ function findNestedGitRepos(absDir: string, relPrefix: string): string[] { /** * Workspace-scope ignore matcher. Ordinary paths get the root's matcher - * (built-in defaults + root `.gitignore`); paths inside an EMBEDDED repo get - * that repo's own matcher (defaults + its root `.gitignore`) — the parent's - * `.gitignore` hides a child repo from git, not from the index (#514). A - * directory path (trailing slash) that is an ANCESTOR of an embedded root is - * never ignored, so directory-pruning callers (the Linux per-directory - * watcher) still descend to reach the embedded repos. + * (built-in defaults + root `.gitignore` + `.git/info/exclude` + + * `core.excludesFile`, plus directories `git ls-files --exclude-standard` + * reports as ignored); paths inside an EMBEDDED repo get that repo's own + * matcher — the parent's `.gitignore` hides a child repo from git, not from + * the index (#514). A directory path (trailing slash) that is an ANCESTOR of + * an embedded root is never ignored, so directory-pruning callers (the Linux + * per-directory watcher) still descend to reach the embedded repos. * - * Single source of truth for indexer and watcher scope — they must not diverge. + * Shared by the indexer (scoped sync / skip checks) and the watcher so their + * scope cannot diverge from each other or from `git ls-files --exclude-standard` + * (#1728). */ export class ScopeIgnore { private embedded: Array<{ root: string; matcher: Ignore }>; @@ -710,8 +813,15 @@ export class ScopeIgnore { export function buildScopeIgnore(rootDir: string, embeddedRoots?: Iterable): ScopeIgnore { const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir); const include = loadIncludeMatcher(rootDir); + // Root matcher already has defaults + root `.gitignore` + info/exclude + + // core.excludesFile. Seed ignored-untracked directories from git so nested + // `.gitignore` effects prune the watcher identically to the indexer (#1728). + const rootMatcher = buildDefaultIgnore(rootDir); + for (const dir of listGitIgnoredDirectories(rootDir)) { + rootMatcher.add(dir); + } return new ScopeIgnore( - buildDefaultIgnore(rootDir), + rootMatcher, roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })), loadExcludeMatcher(rootDir), include, @@ -1492,10 +1602,19 @@ export class ExtractionOrchestrator { * same lifecycle the watcher's own matcher already has. */ private scopedSyncMatcher(): ScopeIgnore { - const key = [PROJECT_CONFIG_FILENAME, '.gitignore'] + // Bust when any root-level exclude source the matcher reads may have + // changed. Nested `.gitignore` edits force a full watcher sync, which + // clears this cache (see the full-reconcile branch in sync()). + const gitDir = resolveGitDir(this.rootDir); + const key = [ + PROJECT_CONFIG_FILENAME, + '.gitignore', + gitDir ? path.join(gitDir, 'info', 'exclude') : '', + ] .map((name) => { + if (!name) return '-'; try { - return String(fs.statSync(path.join(this.rootDir, name)).mtimeMs); + return String(fs.statSync(path.isAbsolute(name) ? name : path.join(this.rootDir, name)).mtimeMs); } catch { return '-'; } @@ -2784,6 +2903,10 @@ export class ExtractionOrchestrator { filesChecked = unique.length; if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-scoped: ${Date.now() - tSyncScan}ms (${unique.length} paths, ${trackedFiles.length} tracked)`); } else { + // Full reconcile: drop the memoized scope matcher so a nested + // `.gitignore` / exclude-standard change that forced this full sync is + // visible to the next scoped sync (#1728). + this.scopedMatcher = null; currentFiles = await scanDirectoryAsync(this.rootDir); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-scan: ${Date.now() - tSyncScan}ms (${currentFiles.length} files)`); filesChecked = currentFiles.length; diff --git a/src/sync/watcher.ts b/src/sync/watcher.ts index 034be85..5ea6436 100644 --- a/src/sync/watcher.ts +++ b/src/sync/watcher.ts @@ -24,11 +24,12 @@ * per-file watches are never needed. * * Excluded trees (node_modules/, dist/, .git/, …) are filtered via the - * indexer's `buildScopeIgnore` (built-in default-ignore dirs + the project's - * .gitignore) — on Linux they're never descended into (so they cost no watch), - * and on macOS/Windows the single recursive stream still covers them but their + * indexer's `buildScopeIgnore` (built-in defaults + root `.gitignore` + + * `.git/info/exclude` + `core.excludesFile` + git ignored-untracked dirs) — + * on Linux they're never descended into (so they cost no watch), and on + * macOS/Windows the single recursive stream still covers them but their * events are dropped before any sync is scheduled. Either way the watcher's - * scope matches the indexer's (#276 / #407). + * scope matches `git ls-files --exclude-standard` (#276 / #407 / #1728). */ import * as fs from 'fs'; @@ -328,12 +329,14 @@ export class FileWatcher { * deterministically gate on watcher readiness. */ private readyWaiters: Array<() => void> = []; - // The shared scope matcher (built-in defaults + project .gitignore + the - // `codegraph.json` exclude/include rules, with embedded child repos matched - // by their OWN rules — #514), built at start() and REBUILT whenever one of - // the files it is derived from changes (see `refreshScope`, #1590). Same - // source of truth the indexer uses, so watcher scope can never diverge from - // index scope. An embedded repo created after start() joins the scope on + // The shared scope matcher from `buildScopeIgnore` (built-in defaults + + // root `.gitignore` + `.git/info/exclude` + `core.excludesFile` + dirs + // `git ls-files --exclude-standard` reports ignored + `codegraph.json` + // exclude/include, with embedded child repos matched by their OWN rules — + // #514), built at start() and REBUILT whenever one of the files it is + // derived from changes (see `refreshScope`, #1590). Same construction the + // indexer uses for scoped sync, so watcher scope cannot diverge from index + // scope (#1728). An embedded repo created after start() joins the scope on // the next scope refresh / watcher restart / re-index. private ignoreMatcher: ScopeIgnore | null = null; @@ -574,6 +577,14 @@ export class FileWatcher { */ private handleChange(rel: string): void { if (!rel || rel === '.' || rel.startsWith('..')) return; + // `.git/info/exclude` is otherwise always-ignored with the rest of `.git/`, + // but it feeds `buildScopeIgnore` — allow it through as a scope refresh + // when the platform delivers the event (recursive watchers may; Linux + // per-directory watching does not descend into `.git/`) (#1728). + if (rel === '.git/info/exclude') { + this.refreshScope(rel); + return; + } if (this.isAlwaysIgnored(rel)) return; // The two root files the scope matcher is derived from are handled BEFORE // the matcher is consulted: a user `exclude` pattern that happens to cover