fix(extraction): respect .gitignore by default for embedded-repo discovery (#970, #976) (#980)

#514 (v1.0.0) began walking into gitignored directories to discover and
index the git repos nested inside them. That broke users who rely on
.gitignore to exclude a directory: a gitignored folder of cloned
reference repos blew graphs up (one report went 10k to 500k edges, #976)
and stalled indexing on multi-gigabyte trees of clones (#970).

Respect .gitignore by default again. Discovering embedded repos inside a
gitignored directory is now opt-in via codegraph.json:

    { "includeIgnored": ["packages/", "services/"] }

The single choke point findIgnoredEmbeddedRepos now returns nothing
unless a gitignored dir matches the project's includeIgnored patterns,
and the matcher is threaded from the scan root through the full-index,
incremental-sync, and watcher-scope paths. Downstream ScopeIgnore and the
watcher are unchanged: they key off the discovered embedded roots, so
gating discovery fixes the indexer, sync, and watcher together. Untracked
embedded repos (#193) stay indexed by default.

This restores the super-repo-of-clones behavior (#622, #699) for the
people who want it, while making the default match what every other tool
(and CodeGraph's own git ls-files foundation) does: .gitignore excludes.

project-config.ts now parses codegraph.json once (loadParsedConfig) and
exposes loadIncludeIgnoredPatterns alongside the existing extension map.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/extraction/index.ts

@@ -19,7 +19,7 @@ import {
 import { QueryBuilder } from '../db/queries';
 import { extractFromSource } from './tree-sitter';
 import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages } from './grammars';
-import { loadExtensionOverrides } from '../project-config';
+import { loadExtensionOverrides, loadIncludeIgnoredPatterns } from '../project-config';
 import { isCodeGraphDataDir } from '../directory';
 import { logDebug, logWarn } from '../errors';
 import { validatePathWithinRoot, normalizePath } from '../utils';
@@ -269,6 +269,20 @@ function defaultsOnlyIgnore(): Ignore {
   return ignore().add(DEFAULT_IGNORE_PATTERNS);
 }
 
+/**
+ * Matcher for the project's `codegraph.json` `includeIgnored` patterns — the
+ * explicit opt-in to index embedded git repos living inside gitignored
+ * directories (#622, #699). Returns `null` when the project opted in nothing,
+ * which is the zero-config DEFAULT: `.gitignore` is then fully respected and a
+ * gitignored directory (even one holding nested repos) is never walked or
+ * indexed (#970, #976). Built once per scan/sync/scope operation from the scan
+ * root and threaded down — never global, so multi-project daemons stay isolated.
+ */
+function loadIncludeIgnoredMatcher(rootDir: string): Ignore | null {
+  const patterns = loadIncludeIgnoredPatterns(rootDir);
+  return patterns.length > 0 ? ignore().add(patterns) : null;
+}
+
 /**
  * `git ls-files --directory` collapses a wholly-untracked/ignored directory into
  * one entry — and when the command's own cwd is such a directory (the indexed
@@ -446,9 +460,12 @@ export function buildScopeIgnore(rootDir: string, embeddedRoots?: Iterable<strin
 
 /**
  * Standalone discovery of every embedded repo root under `rootDir` (relative,
- * trailing-slashed) — both the untracked kind (#193) and the gitignored kind
- * (#514), recursively (an embedded repo can embed further repos). Returns []
- * for non-git roots: the filesystem walk handles nested repos there already.
+ * trailing-slashed) — the untracked kind (#193) always, and the gitignored kind
+ * (#514) only for directories the project opted in via `codegraph.json`
+ * `includeIgnored` (#622, #699); otherwise `.gitignore` is respected and they
+ * are not discovered (#970, #976). Recursive (an embedded repo can embed further
+ * repos). Returns [] for non-git roots: the filesystem walk handles nested repos
+ * there already.
  */
 export function discoverEmbeddedRepoRoots(rootDir: string): string[] {
   try {
@@ -458,6 +475,7 @@ export function discoverEmbeddedRepoRoots(rootDir: string): string[] {
   }
   const out: string[] = [];
   const defaults = defaultsOnlyIgnore();
+  const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
   const visit = (repoAbs: string, prefix: string): void => {
     const candidates: string[] = [];
     try {
@@ -472,7 +490,7 @@ export function discoverEmbeddedRepoRoots(rootDir: string): string[] {
         }
       }
     } catch { /* untracked listing failed — ignored-side discovery still runs */ }
-    candidates.push(...findIgnoredEmbeddedRepos(repoAbs));
+    candidates.push(...findIgnoredEmbeddedRepos(repoAbs, includeIgnored, prefix));
     for (const rel of candidates) {
       const full = normalizePath(prefix + rel);
       out.push(full);
@@ -484,15 +502,27 @@ export function discoverEmbeddedRepoRoots(rootDir: string): string[] {
 }
 
 /**
- * Discover embedded repos hidden by `repoDir`'s OWN ignore rules: for each
- * gitignored directory (skipping built-in default excludes), search for nested
- * `.git` roots. Returns repo paths relative to `repoDir`, trailing-slashed.
+ * Discover embedded repos hidden by `repoDir`'s OWN gitignore rules: for each
+ * gitignored directory, search for nested `.git` roots. Returns repo paths
+ * relative to `repoDir`, trailing-slashed.
+ *
+ * OPT-IN ONLY. Walking into a gitignored directory contradicts what every other
+ * tool (and CodeGraph's own `git ls-files` foundation) does — `.gitignore`
+ * excludes. So this returns `[]` unless the project opted the directory in via
+ * `codegraph.json` `includeIgnored`; without that, a gitignored dir — including
+ * a huge reference/data dir full of nested clones — is left untouched (#970,
+ * #976). When opted in, it restores the super-repo-of-clones behavior (#622,
+ * #699). `prefix` is the scan-root-relative path of `repoDir`, so a pattern like
+ * `services/` opts that whole subtree in at any recursion depth. Built-in
+ * default excludes (`node_modules`, …) are always skipped.
  */
-function findIgnoredEmbeddedRepos(repoDir: string): string[] {
+function findIgnoredEmbeddedRepos(repoDir: string, includeIgnored: Ignore | null, prefix: string): string[] {
+  if (!includeIgnored) return [];
   const defaults = defaultsOnlyIgnore();
   const repos: string[] = [];
   for (const dir of listIgnoredDirs(repoDir)) {
     if (defaults.ignores(dir)) continue;
+    if (!includeIgnored.ignores(normalizePath(prefix + dir))) continue;
     repos.push(...findNestedGitRepos(path.join(repoDir, dir), dir));
   }
   return repos;
@@ -509,12 +539,15 @@ function findIgnoredEmbeddedRepos(repoDir: string): string[] {
  * skips them entirely, and untracked output reports them only as an opaque
  * "subdir/" entry (trailing slash) rather than expanding their files. Each
  * embedded repo is its own git boundary, so we re-run `git ls-files` inside it.
- * (See issue #193.) GITIGNORED embedded repos are invisible even to that —
- * they're discovered separately via `findIgnoredEmbeddedRepos` (#514); every
- * embedded repo root (however found) is recorded in `embeddedRoots` so callers
- * can exempt its files from the parent's own gitignore rules.
+ * (See issue #193.) GITIGNORED embedded repos are invisible even to that; they
+ * are discovered separately via `findIgnoredEmbeddedRepos` (#514) but ONLY for
+ * directories the project opted in through `codegraph.json` `includeIgnored`
+ * (`includeIgnored` here, threaded from the scan root) — by default `.gitignore`
+ * is respected and they stay out (#970, #976). Every embedded repo root (however
+ * found) is recorded in `embeddedRoots` so callers can exempt its files from the
+ * parent's own gitignore rules.
  */
-function collectGitFiles(repoDir: string, prefix: string, files: Set<string>, embeddedRoots?: Set<string>): void {
+function collectGitFiles(repoDir: string, prefix: string, files: Set<string>, embeddedRoots?: Set<string>, includeIgnored: Ignore | null = null): void {
   const gitOpts = { cwd: repoDir, encoding: 'utf-8' as const, timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] as ['pipe', 'pipe', 'pipe'], windowsHide: true };
 
   // Tracked files. --recurse-submodules pulls in files from active submodules,
@@ -548,7 +581,7 @@ function collectGitFiles(repoDir: string, prefix: string, files: Set<string>, em
       // it's a duplicate working view of an already-indexed repo (#848).
       if (classifyGitDir(childDir) === 'embedded' && !defaultsOnlyIgnore().ignores(rel)) {
         embeddedRoots?.add(normalizePath(prefix + rel));
-        collectGitFiles(childDir, prefix + rel, files, embeddedRoots);
+        collectGitFiles(childDir, prefix + rel, files, embeddedRoots, includeIgnored);
       }
       continue;
     }
@@ -556,11 +589,13 @@ function collectGitFiles(repoDir: string, prefix: string, files: Set<string>, em
   }
 
   // Embedded repos hidden by THIS repo's ignore rules (`/packages/` in a
-  // super-repo .gitignore) never appear in any listing above — discover and
-  // recurse into them too. (#514)
-  for (const rel of findIgnoredEmbeddedRepos(repoDir)) {
+  // super-repo .gitignore) never appear in any listing above. By default they
+  // stay hidden — `.gitignore` is respected (#970, #976). They are recursed into
+  // only when the project opted the directory in via `codegraph.json`
+  // `includeIgnored` (#622, #699), which `findIgnoredEmbeddedRepos` enforces.
+  for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
     embeddedRoots?.add(normalizePath(prefix + rel));
-    collectGitFiles(path.join(repoDir, rel), prefix + rel, files, embeddedRoots);
+    collectGitFiles(path.join(repoDir, rel), prefix + rel, files, embeddedRoots, includeIgnored);
   }
 }
 
@@ -598,7 +633,7 @@ function getGitVisibleFiles(rootDir: string): Set<string> | null {
 
     const files = new Set<string>();
     const embeddedRoots = new Set<string>();
-    collectGitFiles(rootDir, '', files, embeddedRoots);
+    collectGitFiles(rootDir, '', files, embeddedRoots, loadIncludeIgnoredMatcher(rootDir));
     // Apply built-in default ignores uniformly — to tracked files too, since
     // committing a dependency/build dir doesn't make it project code. A
     // `.gitignore` negation (e.g. `!vendor/`) is the explicit opt-in. (issue #407)
@@ -627,13 +662,15 @@ interface GitChanges {
  * Use `git status` to detect changed files instead of scanning every file.
  * Returns null on failure so callers fall back to full scan.
  *
- * Recurses into embedded repos — both the untracked kind (#193: the parent's
- * status collapses them to an opaque `?? subdir/` entry) and the gitignored
- * kind (#514: they never appear in the parent's status at all) — running
- * `git status` inside each, so changes in a multi-repo workspace sync without
- * a full rescan. Deleting an ENTIRE embedded repo dir is the one case this
- * cannot see (the child status that would report the deletions is gone with
- * it); a full `codegraph index` reconciles that.
+ * Recurses into embedded repos — the untracked kind (#193: the parent's status
+ * collapses them to an opaque `?? subdir/` entry) always, and the gitignored
+ * kind (#514: they never appear in the parent's status at all) only for
+ * directories opted in via `codegraph.json` `includeIgnored` (#622, #699) —
+ * running `git status` inside each, so changes in a multi-repo workspace sync
+ * without a full rescan. By default a gitignored dir is left alone, matching the
+ * full-index scan (#970, #976). Deleting an ENTIRE embedded repo dir is the one
+ * case this cannot see (the child status that would report the deletions is gone
+ * with it); a full `codegraph index` reconciles that.
  */
 function getGitChangedFiles(rootDir: string): GitChanges | null {
   try {
@@ -641,14 +678,14 @@ function getGitChangedFiles(rootDir: string): GitChanges | null {
     // Custom extension → language overrides from the project's codegraph.json,
     // so change detection sees the same custom-extension files the full index does.
     const overrides = loadExtensionOverrides(rootDir);
-    collectGitStatus(rootDir, '', changes, overrides);
+    collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir));
     return changes;
   } catch {
     return null;
   }
 }
 
-function collectGitStatus(repoDir: string, prefix: string, out: GitChanges, overrides?: Record<string, Language>): void {
+function collectGitStatus(repoDir: string, prefix: string, out: GitChanges, overrides?: Record<string, Language>, includeIgnored: Ignore | null = null): void {
   const output = execFileSync(
     'git',
     ['status', '--porcelain', '--no-renames'],
@@ -705,14 +742,16 @@ function collectGitStatus(repoDir: string, prefix: string, out: GitChanges, over
   }
 
   // Recurse embedded repos found under untracked dirs (at the dir itself or
-  // nested deeper) and under this repo's gitignored dirs.
+  // nested deeper). Gitignored dirs are walked only for the directories the
+  // project opted in via `includeIgnored`; by default `.gitignore` is respected
+  // and they are left alone (#970, #976), mirroring the full-index scan.
   for (const rel of untrackedDirs) {
     for (const repoRel of findNestedGitRepos(path.join(repoDir, rel), rel)) {
-      collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides);
+      collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides, includeIgnored);
     }
   }
-  for (const rel of findIgnoredEmbeddedRepos(repoDir)) {
-    collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides);
+  for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
+    collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides, includeIgnored);
   }
 }
 

src/project-config.ts

@@ -34,11 +34,25 @@ export const PROJECT_CONFIG_FILENAME = 'codegraph.json';
 export interface ProjectConfig {
   /** Map of custom file extension (`.foo`) to a supported language id. */
   extensions?: Record<string, string>;
+  /**
+   * Gitignore-style patterns naming gitignored directories whose embedded git
+   * repositories should be indexed anyway — the explicit opt-in to override
+   * `.gitignore` for nested-repo discovery (#622, #699). Absent/empty (the
+   * default) means `.gitignore` is fully respected: gitignored embedded repos
+   * are never discovered or indexed (#970, #976).
+   */
+  includeIgnored?: string[];
+}
+
+/** Parsed, validated view of a project's `codegraph.json`. */
+interface ParsedConfig {
+  extensions: Record<string, Language>;
+  includeIgnored: string[];
 }
 
 interface CacheEntry {
   mtimeMs: number;
-  overrides: Record<string, Language>;
+  config: ParsedConfig;
 }
 
 /**
@@ -47,11 +61,14 @@ interface CacheEntry {
  * `stat` while a single `codegraph.json` is in force. Keying by root keeps two
  * projects in the same process (the daemon / multi-project MCP server) isolated.
  */
-const overridesCache = new Map<string, Record<string, Language>>();
-const cacheMeta = new Map<string, CacheEntry>();
+const cache = new Map<string, CacheEntry>();
 
-/** Shared frozen empty map so the no-config path allocates nothing. */
-const EMPTY: Record<string, Language> = Object.freeze({});
+/** Shared frozen empties so the no-config path allocates nothing. */
+const EMPTY_EXTENSIONS: Record<string, Language> = Object.freeze({});
+const EMPTY_CONFIG: ParsedConfig = Object.freeze({
+  extensions: EMPTY_EXTENSIONS,
+  includeIgnored: Object.freeze([]) as unknown as string[],
+});
 
 /**
  * Normalize a user-provided extension key to the `.ext` lowercase form used by
@@ -74,16 +91,16 @@ function normalizeExtKey(raw: string): string | null {
 }
 
 /**
- * Parse and validate the `extensions` map out of a `codegraph.json` file.
- * Every failure mode degrades to "no overrides from this entry" — a bad file or
- * a typo'd language never throws.
+ * Read + JSON-parse a `codegraph.json` once and return its validated view.
+ * Every failure mode degrades to the zero-config default — a missing file, bad
+ * JSON, or a typo'd value never throws.
  */
-function parseExtensionOverrides(file: string): Record<string, Language> {
+function parseConfig(file: string): ParsedConfig {
   let raw: string;
   try {
     raw = fs.readFileSync(file, 'utf-8');
   } catch {
-    return EMPTY;
+    return EMPTY_CONFIG;
   }
 
   let parsed: unknown;
@@ -94,12 +111,24 @@ function parseExtensionOverrides(file: string): Record<string, Language> {
       file,
       error: err instanceof Error ? err.message : String(err),
     });
-    return EMPTY;
+    return EMPTY_CONFIG;
   }
 
-  if (!parsed || typeof parsed !== 'object') return EMPTY;
+  if (!parsed || typeof parsed !== 'object') return EMPTY_CONFIG;
+
+  const extensions = extractExtensions(parsed, file);
+  const includeIgnored = extractIncludeIgnored(parsed, file);
+  if (extensions === EMPTY_EXTENSIONS && includeIgnored.length === 0) return EMPTY_CONFIG;
+  return { extensions, includeIgnored };
+}
+
+/**
+ * Validate the `extensions` map. Every failure mode degrades to "no overrides
+ * from this entry" — a bad value or a typo'd language never throws.
+ */
+function extractExtensions(parsed: object, file: string): Record<string, Language> {
   const exts = (parsed as ProjectConfig).extensions;
-  if (!exts || typeof exts !== 'object' || Array.isArray(exts)) return EMPTY;
+  if (!exts || typeof exts !== 'object' || Array.isArray(exts)) return EMPTY_EXTENSIONS;
 
   const out: Record<string, Language> = {};
   for (const [rawKey, rawVal] of Object.entries(exts)) {
@@ -115,7 +144,57 @@ function parseExtensionOverrides(file: string): Record<string, Language> {
     out[key] = rawVal as Language;
   }
 
-  return Object.keys(out).length > 0 ? out : EMPTY;
+  return Object.keys(out).length > 0 ? out : EMPTY_EXTENSIONS;
+}
+
+/**
+ * Validate the `includeIgnored` patterns: an array of non-empty gitignore-style
+ * strings. A non-array value or a non-string/blank entry warns-and-skips; never
+ * throws. Patterns are kept verbatim (trimmed) so they match exactly as a
+ * `.gitignore` line would.
+ */
+function extractIncludeIgnored(parsed: object, file: string): string[] {
+  const raw = (parsed as ProjectConfig).includeIgnored;
+  if (raw === undefined) return [];
+  if (!Array.isArray(raw)) {
+    logWarn(`Ignoring "includeIgnored" in ${PROJECT_CONFIG_FILENAME}: must be an array of gitignore-style patterns`, { file });
+    return [];
+  }
+
+  const out: string[] = [];
+  for (const entry of raw) {
+    if (typeof entry !== 'string' || !entry.trim()) {
+      logWarn(`Ignoring an "includeIgnored" entry in ${PROJECT_CONFIG_FILENAME}: every pattern must be a non-empty string`, { file });
+      continue;
+    }
+    out.push(entry.trim());
+  }
+  return out;
+}
+
+/**
+ * Load the parsed `codegraph.json` for a project, mtime-cached. A missing or
+ * malformed file yields the zero-config default. One `stat` (and at most one
+ * read/parse) while a single config file is in force, shared across every field.
+ */
+function loadParsedConfig(rootDir: string): ParsedConfig {
+  const file = path.join(rootDir, PROJECT_CONFIG_FILENAME);
+
+  let mtimeMs: number;
+  try {
+    mtimeMs = fs.statSync(file).mtimeMs;
+  } catch {
+    // No config file — drop any stale cache entry and return the default.
+    cache.delete(rootDir);
+    return EMPTY_CONFIG;
+  }
+
+  const entry = cache.get(rootDir);
+  if (entry && entry.mtimeMs === mtimeMs) return entry.config;
+
+  const config = parseConfig(file);
+  cache.set(rootDir, { mtimeMs, config });
+  return config;
 }
 
 /**
@@ -127,29 +206,22 @@ function parseExtensionOverrides(file: string): Record<string, Language> {
  * map when there is no `codegraph.json` (the zero-config default).
  */
 export function loadExtensionOverrides(rootDir: string): Record<string, Language> {
-  const file = path.join(rootDir, PROJECT_CONFIG_FILENAME);
+  return loadParsedConfig(rootDir).extensions;
+}
 
-  let mtimeMs: number;
-  try {
-    mtimeMs = fs.statSync(file).mtimeMs;
-  } catch {
-    // No config file — drop any stale cache entry and return the default.
-    cacheMeta.delete(rootDir);
-    overridesCache.delete(rootDir);
-    return EMPTY;
-  }
-
-  const meta = cacheMeta.get(rootDir);
-  if (meta && meta.mtimeMs === mtimeMs) return meta.overrides;
-
-  const overrides = parseExtensionOverrides(file);
-  cacheMeta.set(rootDir, { mtimeMs, overrides });
-  overridesCache.set(rootDir, overrides);
-  return overrides;
+/**
+ * Load the validated `includeIgnored` patterns for a project, mtime-cached.
+ *
+ * These name gitignored directories whose embedded git repositories should be
+ * indexed despite `.gitignore` (#622, #699). An empty result — the zero-config
+ * default — means `.gitignore` is fully respected: gitignored embedded repos
+ * are never discovered or indexed (#970, #976).
+ */
+export function loadIncludeIgnoredPatterns(rootDir: string): string[] {
+  return loadParsedConfig(rootDir).includeIgnored;
 }
 
 /** Test/maintenance hook: forget cached config (e.g. after rewriting it in a test). */
 export function clearProjectConfigCache(): void {
-  cacheMeta.clear();
-  overridesCache.clear();
+  cache.clear();
 }