fix(extraction): respect .gitignore by default for embedded-repo discovery (#970, #976) (#980)

#514 (v1.0.0) began walking into gitignored directories to discover and
index the git repos nested inside them. That broke users who rely on
.gitignore to exclude a directory: a gitignored folder of cloned
reference repos blew graphs up (one report went 10k to 500k edges, #976)
and stalled indexing on multi-gigabyte trees of clones (#970).

Respect .gitignore by default again. Discovering embedded repos inside a
gitignored directory is now opt-in via codegraph.json:

    { "includeIgnored": ["packages/", "services/"] }

The single choke point findIgnoredEmbeddedRepos now returns nothing
unless a gitignored dir matches the project's includeIgnored patterns,
and the matcher is threaded from the scan root through the full-index,
incremental-sync, and watcher-scope paths. Downstream ScopeIgnore and the
watcher are unchanged: they key off the discovered embedded roots, so
gating discovery fixes the indexer, sync, and watcher together. Untracked
embedded repos (#193) stay indexed by default.

This restores the super-repo-of-clones behavior (#622, #699) for the
people who want it, while making the default match what every other tool
(and CodeGraph's own git ls-files foundation) does: .gitignore excludes.

project-config.ts now parses codegraph.json once (loadParsedConfig) and
exposes loadIncludeIgnoredPatterns alongside the existing extension map.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/project-config.ts

@@ -34,11 +34,25 @@ export const PROJECT_CONFIG_FILENAME = 'codegraph.json';
 export interface ProjectConfig {
   /** Map of custom file extension (`.foo`) to a supported language id. */
   extensions?: Record<string, string>;
+  /**
+   * Gitignore-style patterns naming gitignored directories whose embedded git
+   * repositories should be indexed anyway — the explicit opt-in to override
+   * `.gitignore` for nested-repo discovery (#622, #699). Absent/empty (the
+   * default) means `.gitignore` is fully respected: gitignored embedded repos
+   * are never discovered or indexed (#970, #976).
+   */
+  includeIgnored?: string[];
+}
+
+/** Parsed, validated view of a project's `codegraph.json`. */
+interface ParsedConfig {
+  extensions: Record<string, Language>;
+  includeIgnored: string[];
 }
 
 interface CacheEntry {
   mtimeMs: number;
-  overrides: Record<string, Language>;
+  config: ParsedConfig;
 }
 
 /**
@@ -47,11 +61,14 @@ interface CacheEntry {
  * `stat` while a single `codegraph.json` is in force. Keying by root keeps two
  * projects in the same process (the daemon / multi-project MCP server) isolated.
  */
-const overridesCache = new Map<string, Record<string, Language>>();
-const cacheMeta = new Map<string, CacheEntry>();
+const cache = new Map<string, CacheEntry>();
 
-/** Shared frozen empty map so the no-config path allocates nothing. */
-const EMPTY: Record<string, Language> = Object.freeze({});
+/** Shared frozen empties so the no-config path allocates nothing. */
+const EMPTY_EXTENSIONS: Record<string, Language> = Object.freeze({});
+const EMPTY_CONFIG: ParsedConfig = Object.freeze({
+  extensions: EMPTY_EXTENSIONS,
+  includeIgnored: Object.freeze([]) as unknown as string[],
+});
 
 /**
  * Normalize a user-provided extension key to the `.ext` lowercase form used by
@@ -74,16 +91,16 @@ function normalizeExtKey(raw: string): string | null {
 }
 
 /**
- * Parse and validate the `extensions` map out of a `codegraph.json` file.
- * Every failure mode degrades to "no overrides from this entry" — a bad file or
- * a typo'd language never throws.
+ * Read + JSON-parse a `codegraph.json` once and return its validated view.
+ * Every failure mode degrades to the zero-config default — a missing file, bad
+ * JSON, or a typo'd value never throws.
  */
-function parseExtensionOverrides(file: string): Record<string, Language> {
+function parseConfig(file: string): ParsedConfig {
   let raw: string;
   try {
     raw = fs.readFileSync(file, 'utf-8');
   } catch {
-    return EMPTY;
+    return EMPTY_CONFIG;
   }
 
   let parsed: unknown;
@@ -94,12 +111,24 @@ function parseExtensionOverrides(file: string): Record<string, Language> {
       file,
       error: err instanceof Error ? err.message : String(err),
     });
-    return EMPTY;
+    return EMPTY_CONFIG;
   }
 
-  if (!parsed || typeof parsed !== 'object') return EMPTY;
+  if (!parsed || typeof parsed !== 'object') return EMPTY_CONFIG;
+
+  const extensions = extractExtensions(parsed, file);
+  const includeIgnored = extractIncludeIgnored(parsed, file);
+  if (extensions === EMPTY_EXTENSIONS && includeIgnored.length === 0) return EMPTY_CONFIG;
+  return { extensions, includeIgnored };
+}
+
+/**
+ * Validate the `extensions` map. Every failure mode degrades to "no overrides
+ * from this entry" — a bad value or a typo'd language never throws.
+ */
+function extractExtensions(parsed: object, file: string): Record<string, Language> {
   const exts = (parsed as ProjectConfig).extensions;
-  if (!exts || typeof exts !== 'object' || Array.isArray(exts)) return EMPTY;
+  if (!exts || typeof exts !== 'object' || Array.isArray(exts)) return EMPTY_EXTENSIONS;
 
   const out: Record<string, Language> = {};
   for (const [rawKey, rawVal] of Object.entries(exts)) {
@@ -115,7 +144,57 @@ function parseExtensionOverrides(file: string): Record<string, Language> {
     out[key] = rawVal as Language;
   }
 
-  return Object.keys(out).length > 0 ? out : EMPTY;
+  return Object.keys(out).length > 0 ? out : EMPTY_EXTENSIONS;
+}
+
+/**
+ * Validate the `includeIgnored` patterns: an array of non-empty gitignore-style
+ * strings. A non-array value or a non-string/blank entry warns-and-skips; never
+ * throws. Patterns are kept verbatim (trimmed) so they match exactly as a
+ * `.gitignore` line would.
+ */
+function extractIncludeIgnored(parsed: object, file: string): string[] {
+  const raw = (parsed as ProjectConfig).includeIgnored;
+  if (raw === undefined) return [];
+  if (!Array.isArray(raw)) {
+    logWarn(`Ignoring "includeIgnored" in ${PROJECT_CONFIG_FILENAME}: must be an array of gitignore-style patterns`, { file });
+    return [];
+  }
+
+  const out: string[] = [];
+  for (const entry of raw) {
+    if (typeof entry !== 'string' || !entry.trim()) {
+      logWarn(`Ignoring an "includeIgnored" entry in ${PROJECT_CONFIG_FILENAME}: every pattern must be a non-empty string`, { file });
+      continue;
+    }
+    out.push(entry.trim());
+  }
+  return out;
+}
+
+/**
+ * Load the parsed `codegraph.json` for a project, mtime-cached. A missing or
+ * malformed file yields the zero-config default. One `stat` (and at most one
+ * read/parse) while a single config file is in force, shared across every field.
+ */
+function loadParsedConfig(rootDir: string): ParsedConfig {
+  const file = path.join(rootDir, PROJECT_CONFIG_FILENAME);
+
+  let mtimeMs: number;
+  try {
+    mtimeMs = fs.statSync(file).mtimeMs;
+  } catch {
+    // No config file — drop any stale cache entry and return the default.
+    cache.delete(rootDir);
+    return EMPTY_CONFIG;
+  }
+
+  const entry = cache.get(rootDir);
+  if (entry && entry.mtimeMs === mtimeMs) return entry.config;
+
+  const config = parseConfig(file);
+  cache.set(rootDir, { mtimeMs, config });
+  return config;
 }
 
 /**
@@ -127,29 +206,22 @@ function parseExtensionOverrides(file: string): Record<string, Language> {
  * map when there is no `codegraph.json` (the zero-config default).
  */
 export function loadExtensionOverrides(rootDir: string): Record<string, Language> {
-  const file = path.join(rootDir, PROJECT_CONFIG_FILENAME);
+  return loadParsedConfig(rootDir).extensions;
+}
 
-  let mtimeMs: number;
-  try {
-    mtimeMs = fs.statSync(file).mtimeMs;
-  } catch {
-    // No config file — drop any stale cache entry and return the default.
-    cacheMeta.delete(rootDir);
-    overridesCache.delete(rootDir);
-    return EMPTY;
-  }
-
-  const meta = cacheMeta.get(rootDir);
-  if (meta && meta.mtimeMs === mtimeMs) return meta.overrides;
-
-  const overrides = parseExtensionOverrides(file);
-  cacheMeta.set(rootDir, { mtimeMs, overrides });
-  overridesCache.set(rootDir, overrides);
-  return overrides;
+/**
+ * Load the validated `includeIgnored` patterns for a project, mtime-cached.
+ *
+ * These name gitignored directories whose embedded git repositories should be
+ * indexed despite `.gitignore` (#622, #699). An empty result — the zero-config
+ * default — means `.gitignore` is fully respected: gitignored embedded repos
+ * are never discovered or indexed (#970, #976).
+ */
+export function loadIncludeIgnoredPatterns(rootDir: string): string[] {
+  return loadParsedConfig(rootDir).includeIgnored;
 }
 
 /** Test/maintenance hook: forget cached config (e.g. after rewriting it in a test). */
 export function clearProjectConfigCache(): void {
-  cacheMeta.clear();
-  overridesCache.clear();
+  cache.clear();
 }