feat(config): map custom file extensions to languages via codegraph.json (#906) (#955)

The extension → language table was hardcoded, so a codebase using a
non-standard extension for a supported language (e.g. `.dota_lua` for Lua)
had those files silently skipped — no way to opt them in short of patching
the source.

Add an opt-in, project-scoped `codegraph.json` at the repo root:

    { "extensions": { ".dota_lua": "lua", ".tpl": "php" } }

Mappings merge on top of the built-in defaults and take precedence (so a
built-in can be re-pointed, e.g. `.h` → `cpp`). Absent or malformed config
is the zero-config default — byte-identical to prior behavior; an invalid
target language or unparseable file is warned-and-skipped, never fatal.

Implementation:
- New `src/project-config.ts` — `loadExtensionOverrides(rootDir)`, validated
  against `isLanguageSupported`, mtime-cached per root.
- `detectLanguage` / `isSourceFile` gain an optional `overrides` arg
  (omitting it is the existing behavior).
- Overrides threaded per-operation through every extraction call site
  (scan/walk gates, git change-detection, grammar selection, extraction,
  the file watcher), resolved from the project root — no process-global
  state, so the multi-project daemon stays isolated. The parse worker
  receives the resolved language in its message.

Tests: 13 new cases (unit, loader validation/normalization/caching, and a
full-index integration proving a custom-extension file is extracted while
the zero-config path indexes nothing). Worker path smoke-tested via the
built CLI.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/extraction/grammars.ts

@@ -121,13 +121,18 @@ export const EXTENSION_MAP: Record<string, Language> = {
  * Whether a file is one CodeGraph can parse, based purely on its extension.
  * This is the single source of truth for "should we index this file" — derived
  * from EXTENSION_MAP so parser support and indexing selection never drift.
+ *
+ * `overrides` is the project's validated custom extension → language map (from
+ * `codegraph.json`); when present its extensions count as indexable in addition
+ * to the built-ins. Omitting it is byte-identical to the zero-config behavior.
  */
-export function isSourceFile(filePath: string): boolean {
+export function isSourceFile(filePath: string, overrides?: Record<string, Language>): boolean {
   if (isPlayRoutesFile(filePath)) return true; // Play `conf/routes` is extensionless
   if (isShopifyLiquidJson(filePath)) return true; // Shopify OS 2.0 JSON templates / section groups
   const dot = filePath.lastIndexOf('.');
   if (dot < 0) return false;
-  return filePath.slice(dot).toLowerCase() in EXTENSION_MAP;
+  const ext = filePath.slice(dot).toLowerCase();
+  return ext in EXTENSION_MAP || (!!overrides && ext in overrides);
 }
 
 /**
@@ -266,9 +271,13 @@ export function getParser(language: Language): Parser | null {
 }
 
 /**
- * Detect language from file extension
+ * Detect language from file extension.
+ *
+ * `overrides` is the project's validated custom extension → language map (from
+ * `codegraph.json`); when present its mappings take precedence over the built-in
+ * `EXTENSION_MAP`. Omitting it is byte-identical to the zero-config behavior.
  */
-export function detectLanguage(filePath: string, source?: string): Language {
+export function detectLanguage(filePath: string, source?: string, overrides?: Record<string, Language>): Language {
   // Play `conf/routes` has no grammar — route through the no-symbol path; the
   // Play framework resolver extracts route nodes from it.
   if (isPlayRoutesFile(filePath)) return 'yaml';
@@ -276,7 +285,7 @@ export function detectLanguage(filePath: string, source?: string): Language {
   // Shopify OS 2.0 JSON templates / section groups → the Liquid extractor (it
   // links each section `"type"` to its `sections/<type>.liquid`).
   if (isShopifyLiquidJson(filePath)) return 'liquid';
-  const lang = EXTENSION_MAP[ext] || 'unknown';
+  const lang = (overrides && overrides[ext]) || EXTENSION_MAP[ext] || 'unknown';
 
   // .h files could be C, C++, or Objective-C — check source content
   if (lang === 'c' && ext === '.h' && source) {

src/extraction/index.ts

@@ -19,6 +19,7 @@ import {
 import { QueryBuilder } from '../db/queries';
 import { extractFromSource } from './tree-sitter';
 import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages } from './grammars';
+import { loadExtensionOverrides } from '../project-config';
 import { isCodeGraphDataDir } from '../directory';
 import { logDebug, logWarn } from '../errors';
 import { validatePathWithinRoot, normalizePath } from '../utils';
@@ -637,14 +638,17 @@ interface GitChanges {
 function getGitChangedFiles(rootDir: string): GitChanges | null {
   try {
     const changes: GitChanges = { modified: [], added: [], deleted: [] };
-    collectGitStatus(rootDir, '', changes);
+    // Custom extension → language overrides from the project's codegraph.json,
+    // so change detection sees the same custom-extension files the full index does.
+    const overrides = loadExtensionOverrides(rootDir);
+    collectGitStatus(rootDir, '', changes, overrides);
     return changes;
   } catch {
     return null;
   }
 }
 
-function collectGitStatus(repoDir: string, prefix: string, out: GitChanges): void {
+function collectGitStatus(repoDir: string, prefix: string, out: GitChanges, overrides?: Record<string, Language>): void {
   const output = execFileSync(
     'git',
     ['status', '--porcelain', '--no-renames'],
@@ -678,7 +682,7 @@ function collectGitStatus(repoDir: string, prefix: string, out: GitChanges): voi
     }
 
     const filePath = normalizePath(prefix + rel);
-    if (!isSourceFile(filePath)) continue;
+    if (!isSourceFile(filePath, overrides)) continue;
 
     if (statusCode.includes('D')) {
       // Deletions stay unfiltered: getChangedFiles acts on one only when the
@@ -704,11 +708,11 @@ function collectGitStatus(repoDir: string, prefix: string, out: GitChanges): voi
   // nested deeper) and under this repo's gitignored dirs.
   for (const rel of untrackedDirs) {
     for (const repoRel of findNestedGitRepos(path.join(repoDir, rel), rel)) {
-      collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out);
+      collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides);
     }
   }
   for (const rel of findIgnoredEmbeddedRepos(repoDir)) {
-    collectGitStatus(path.join(repoDir, rel), prefix + rel, out);
+    collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides);
   }
 }
 
@@ -723,13 +727,16 @@ export function scanDirectory(
   rootDir: string,
   onProgress?: (current: number, file: string) => void
 ): string[] {
+  // Custom extension → language overrides from the project's codegraph.json.
+  const overrides = loadExtensionOverrides(rootDir);
+
   // Fast path: use git to get all visible files (respects .gitignore everywhere)
   const gitFiles = getGitVisibleFiles(rootDir);
   if (gitFiles) {
     const files: string[] = [];
     let count = 0;
     for (const filePath of gitFiles) {
-      if (isSourceFile(filePath)) {
+      if (isSourceFile(filePath, overrides)) {
         files.push(filePath);
         count++;
         onProgress?.(count, filePath);
@@ -750,12 +757,15 @@ export async function scanDirectoryAsync(
   rootDir: string,
   onProgress?: (current: number, file: string) => void
 ): Promise<string[]> {
+  // Custom extension → language overrides from the project's codegraph.json.
+  const overrides = loadExtensionOverrides(rootDir);
+
   const gitFiles = getGitVisibleFiles(rootDir);
   if (gitFiles) {
     const files: string[] = [];
     let count = 0;
     for (const filePath of gitFiles) {
-      if (isSourceFile(filePath)) {
+      if (isSourceFile(filePath, overrides)) {
         files.push(filePath);
         count++;
         onProgress?.(count, filePath);
@@ -781,6 +791,8 @@ function scanDirectoryWalk(
   const files: string[] = [];
   let count = 0;
   const visitedDirs = new Set<string>();
+  // Custom extension → language overrides from the project's codegraph.json.
+  const overrides = loadExtensionOverrides(rootDir);
 
   // A .gitignore matcher scoped to the directory that declared it. Patterns in
   // a nested .gitignore are relative to that directory, so we keep the dir
@@ -857,7 +869,7 @@ function scanDirectoryWalk(
               walk(fullPath, active);
             }
           } else if (stat.isFile()) {
-            if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath)) {
+            if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, overrides)) {
               files.push(relativePath);
               count++;
               onProgress?.(count, relativePath);
@@ -874,7 +886,7 @@ function scanDirectoryWalk(
           walk(fullPath, active);
         }
       } else if (entry.isFile()) {
-        if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath)) {
+        if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, overrides)) {
           files.push(relativePath);
           count++;
           onProgress?.(count, relativePath);
@@ -994,6 +1006,11 @@ export class ExtractionOrchestrator {
     let totalNodes = 0;
     let totalEdges = 0;
 
+    // Custom extension → language overrides from the project's codegraph.json.
+    // Threaded into language detection so custom-extension files load the right
+    // grammar and store under the mapped language.
+    const overrides = loadExtensionOverrides(this.rootDir);
+
     const log = verbose
       ? (msg: string) => { console.log(`[worker] ${msg}`); }
       : (_msg: string) => {};
@@ -1050,7 +1067,7 @@ export class ExtractionOrchestrator {
     await new Promise(resolve => setImmediate(resolve));
 
     // Detect needed languages and load grammars in the parse worker
-    const neededLanguages = [...new Set(files.map((f) => detectLanguage(f)))];
+    const neededLanguages = [...new Set(files.map((f) => detectLanguage(f, undefined, overrides)))];
     // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed
     if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
       neededLanguages.push('cpp');
@@ -1161,12 +1178,17 @@ export class ExtractionOrchestrator {
     }
 
     async function requestParse(filePath: string, content: string): Promise<ExtractionResult> {
+      // Resolve the language on the main thread (where the project's
+      // codegraph.json overrides are loaded) and hand it to the worker, so the
+      // worker never needs the override map itself.
+      const language = detectLanguage(filePath, content, overrides);
+
       if (!WorkerClass) {
         // In-process fallback
         return extractFromSource(
           filePath,
           content,
-          detectLanguage(filePath, content),
+          language,
           frameworkNames
         );
       }
@@ -1198,7 +1220,7 @@ export class ExtractionOrchestrator {
         }, timeoutMs);
 
         pendingParses.set(id, { resolve, reject, timer });
-        worker.postMessage({ type: 'parse', id, filePath, content, frameworkNames });
+        worker.postMessage({ type: 'parse', id, filePath, content, frameworkNames, language });
       });
     }
 
@@ -1312,7 +1334,7 @@ export class ExtractionOrchestrator {
 
         // Store in database on main thread (SQLite is not thread-safe)
         if (result.nodes.length > 0 || result.errors.length === 0) {
-          const language = detectLanguage(filePath, content);
+          const language = detectLanguage(filePath, content, overrides);
           this.storeExtractionResult(filePath, content, language, stats, result);
         }
 
@@ -1333,7 +1355,7 @@ export class ExtractionOrchestrator {
           // Files with no symbols but no errors (yaml, twig, properties) are
           // tracked at the file level — count them as indexed so the CLI
           // doesn't misleadingly report "No files found to index".
-          const lang = detectLanguage(filePath, content);
+          const lang = detectLanguage(filePath, content, overrides);
           if (isFileLevelOnlyLanguage(lang)) {
             filesIndexed++;
           } else {
@@ -1393,7 +1415,7 @@ export class ExtractionOrchestrator {
         }
 
         if (result.nodes.length > 0 || result.errors.length === 0) {
-          const language = detectLanguage(filePath, content);
+          const language = detectLanguage(filePath, content, overrides);
           const stats = await fsp.stat(path.join(this.rootDir, filePath));
           this.storeExtractionResult(filePath, content, language, stats, result);
 
@@ -1444,7 +1466,7 @@ export class ExtractionOrchestrator {
           }
 
           if (result.nodes.length > 0 || result.errors.length === 0) {
-            const language = detectLanguage(filePath, fullContent);
+            const language = detectLanguage(filePath, fullContent, overrides);
             const stats = await fsp.stat(path.join(this.rootDir, filePath));
             this.storeExtractionResult(filePath, fullContent, language, stats, result);
 
@@ -1607,8 +1629,8 @@ export class ExtractionOrchestrator {
       };
     }
 
-    // Detect language
-    const language = detectLanguage(relativePath, content);
+    // Detect language (honoring the project's codegraph.json extension overrides)
+    const language = detectLanguage(relativePath, content, loadExtensionOverrides(this.rootDir));
     if (!isLanguageSupported(language)) {
       return {
         nodes: [],
@@ -1863,7 +1885,8 @@ export class ExtractionOrchestrator {
 
     // Load only grammars needed for changed files
     if (filesToIndex.length > 0) {
-      const neededLanguages = [...new Set(filesToIndex.map((f) => detectLanguage(f)))];
+      const overrides = loadExtensionOverrides(this.rootDir);
+      const neededLanguages = [...new Set(filesToIndex.map((f) => detectLanguage(f, undefined, overrides)))];
       // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded
       if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
         neededLanguages.push('cpp');

src/extraction/parse-worker.ts

@@ -55,14 +55,17 @@ import type { Language, ExtractionResult } from '../types';
 const PARSER_RESET_INTERVAL = 5000;
 const parseCounts = new Map<Language, number>();
 
-parentPort!.on('message', async (msg: { type: string; id?: number; filePath?: string; content?: string; languages?: Language[]; frameworkNames?: string[] }) => {
+parentPort!.on('message', async (msg: { type: string; id?: number; filePath?: string; content?: string; languages?: Language[]; frameworkNames?: string[]; language?: Language }) => {
   if (msg.type === 'load-grammars') {
     await loadGrammarsForLanguages(msg.languages!);
     parentPort!.postMessage({ type: 'grammars-loaded' });
   } else if (msg.type === 'parse') {
     const { id, filePath, content, frameworkNames } = msg;
     try {
-      const language = detectLanguage(filePath!, content);
+      // The main thread resolves the language (it holds the project's
+      // codegraph.json extension overrides) and sends it; fall back to detection
+      // for older callers / safety.
+      const language = msg.language ?? detectLanguage(filePath!, content);
       const result: ExtractionResult = extractFromSource(filePath!, content!, language, frameworkNames);
 
       // Periodic parser reset to reclaim WASM heap memory

src/project-config.ts

@@ -0,0 +1,155 @@
+/**
+ * Project-scoped configuration: a committed `codegraph.json` at the project
+ * root that a team shares through version control.
+ *
+ * Today it carries one thing — `extensions`, an opt-in map from a custom file
+ * extension to one of CodeGraph's supported languages. The built-in
+ * extension → language table (`EXTENSION_MAP` in `extraction/grammars.ts`) is
+ * otherwise hardcoded, so a codebase that uses a non-standard extension for a
+ * supported language (e.g. `.dota_lua` for Lua) sees those files silently
+ * skipped. This lets the project map them once, in a version-controlled file:
+ *
+ *   {
+ *     "extensions": {
+ *       ".dota_lua": "lua",
+ *       ".tpl": "php"
+ *     }
+ *   }
+ *
+ * User mappings merge on TOP of the built-ins and win on conflict, so a project
+ * can also re-point a built-in extension (e.g. force `.h` → `cpp`). Absent or
+ * malformed config is the zero-config default — no overrides, no error. Invalid
+ * individual entries are warned-and-skipped (never fatal): an unparseable
+ * project file must not break indexing.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { Language } from './types';
+import { isLanguageSupported } from './extraction/grammars';
+import { logWarn } from './errors';
+
+/** Filename of the project-scoped config, resolved relative to the project root. */
+export const PROJECT_CONFIG_FILENAME = 'codegraph.json';
+
+export interface ProjectConfig {
+  /** Map of custom file extension (`.foo`) to a supported language id. */
+  extensions?: Record<string, string>;
+}
+
+interface CacheEntry {
+  mtimeMs: number;
+  overrides: Record<string, Language>;
+}
+
+/**
+ * Cache keyed by project root. The loader is called once per indexing/scan/sync
+ * operation (and per watch event), so the mtime guard keeps repeat calls to one
+ * `stat` while a single `codegraph.json` is in force. Keying by root keeps two
+ * projects in the same process (the daemon / multi-project MCP server) isolated.
+ */
+const overridesCache = new Map<string, Record<string, Language>>();
+const cacheMeta = new Map<string, CacheEntry>();
+
+/** Shared frozen empty map so the no-config path allocates nothing. */
+const EMPTY: Record<string, Language> = Object.freeze({});
+
+/**
+ * Normalize a user-provided extension key to the `.ext` lowercase form used by
+ * the built-in map. Returns null for keys that can never match a real file
+ * extension (so the caller warns and skips):
+ *   - empty / just "."
+ *   - multi-part (".d.ts") — language detection keys off the FINAL extension
+ *     only (`lastIndexOf('.')`), so a multi-dot key would never be consulted.
+ *   - anything containing a path separator.
+ */
+function normalizeExtKey(raw: string): string | null {
+  if (typeof raw !== 'string') return null;
+  let ext = raw.trim().toLowerCase();
+  if (!ext) return null;
+  if (!ext.startsWith('.')) ext = '.' + ext;
+  const body = ext.slice(1);
+  if (!body) return null;
+  if (body.includes('.') || body.includes('/') || body.includes('\\')) return null;
+  return ext;
+}
+
+/**
+ * Parse and validate the `extensions` map out of a `codegraph.json` file.
+ * Every failure mode degrades to "no overrides from this entry" — a bad file or
+ * a typo'd language never throws.
+ */
+function parseExtensionOverrides(file: string): Record<string, Language> {
+  let raw: string;
+  try {
+    raw = fs.readFileSync(file, 'utf-8');
+  } catch {
+    return EMPTY;
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    logWarn(`Ignoring ${PROJECT_CONFIG_FILENAME}: not valid JSON`, {
+      file,
+      error: err instanceof Error ? err.message : String(err),
+    });
+    return EMPTY;
+  }
+
+  if (!parsed || typeof parsed !== 'object') return EMPTY;
+  const exts = (parsed as ProjectConfig).extensions;
+  if (!exts || typeof exts !== 'object' || Array.isArray(exts)) return EMPTY;
+
+  const out: Record<string, Language> = {};
+  for (const [rawKey, rawVal] of Object.entries(exts)) {
+    const key = normalizeExtKey(rawKey);
+    if (!key) {
+      logWarn(`Ignoring extension mapping in ${PROJECT_CONFIG_FILENAME}: "${rawKey}" is not a valid file extension`, { file });
+      continue;
+    }
+    if (typeof rawVal !== 'string' || !isLanguageSupported(rawVal as Language)) {
+      logWarn(`Ignoring extension "${rawKey}" in ${PROJECT_CONFIG_FILENAME}: "${String(rawVal)}" is not a supported language`, { file });
+      continue;
+    }
+    out[key] = rawVal as Language;
+  }
+
+  return Object.keys(out).length > 0 ? out : EMPTY;
+}
+
+/**
+ * Load the validated extension overrides for a project, mtime-cached.
+ *
+ * Returns a map of `.ext` → supported language id. The result merges on top of
+ * the built-in extension map at the point of use (see `detectLanguage` /
+ * `isSourceFile`), with these user mappings taking precedence. Returns an empty
+ * map when there is no `codegraph.json` (the zero-config default).
+ */
+export function loadExtensionOverrides(rootDir: string): Record<string, Language> {
+  const file = path.join(rootDir, PROJECT_CONFIG_FILENAME);
+
+  let mtimeMs: number;
+  try {
+    mtimeMs = fs.statSync(file).mtimeMs;
+  } catch {
+    // No config file — drop any stale cache entry and return the default.
+    cacheMeta.delete(rootDir);
+    overridesCache.delete(rootDir);
+    return EMPTY;
+  }
+
+  const meta = cacheMeta.get(rootDir);
+  if (meta && meta.mtimeMs === mtimeMs) return meta.overrides;
+
+  const overrides = parseExtensionOverrides(file);
+  cacheMeta.set(rootDir, { mtimeMs, overrides });
+  overridesCache.set(rootDir, overrides);
+  return overrides;
+}
+
+/** Test/maintenance hook: forget cached config (e.g. after rewriting it in a test). */
+export function clearProjectConfigCache(): void {
+  cacheMeta.clear();
+  overridesCache.clear();
+}

src/sync/watcher.ts

@@ -34,6 +34,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { isSourceFile, buildScopeIgnore, type ScopeIgnore } from '../extraction';
+import { loadExtensionOverrides } from '../project-config';
 import { logDebug, logWarn } from '../errors';
 import { normalizePath } from '../utils';
 import { isCodeGraphDataDir } from '../directory';
@@ -535,7 +536,7 @@ export class FileWatcher {
     if (!rel || rel === '.' || rel.startsWith('..')) return;
     if (this.isAlwaysIgnored(rel)) return;
     if (this.ignoreMatcher && this.ignoreMatcher.ignores(rel)) return;
-    if (!isSourceFile(rel)) return;
+    if (!isSourceFile(rel, loadExtensionOverrides(this.projectRoot))) return;
 
     logDebug('File change detected', { file: rel });
     if (this.ready) {