/** * Grammar Loading and Caching * * Uses web-tree-sitter (WASM) for universal cross-platform support. * Grammars are loaded lazily — only languages actually present in the project * are compiled, keeping V8 WASM memory pressure low on large codebases. */ import * as path from 'path'; import * as fsp from 'fs/promises'; import { Parser, Language as WasmLanguage } from 'web-tree-sitter'; import { Language } from '../types'; export type GrammarLanguage = Exclude; /** * WASM filename map — maps each language to its .wasm grammar file * in the tree-sitter-wasms package. */ const WASM_GRAMMAR_FILES: Record = { typescript: 'tree-sitter-typescript.wasm', tsx: 'tree-sitter-tsx.wasm', javascript: 'tree-sitter-javascript.wasm', jsx: 'tree-sitter-javascript.wasm', python: 'tree-sitter-python.wasm', go: 'tree-sitter-go.wasm', rust: 'tree-sitter-rust.wasm', java: 'tree-sitter-java.wasm', c: 'tree-sitter-c.wasm', cpp: 'tree-sitter-cpp.wasm', csharp: 'tree-sitter-c_sharp.wasm', php: 'tree-sitter-php.wasm', ruby: 'tree-sitter-ruby.wasm', swift: 'tree-sitter-swift.wasm', kotlin: 'tree-sitter-kotlin.wasm', dart: 'tree-sitter-dart.wasm', pascal: 'tree-sitter-pascal.wasm', scala: 'tree-sitter-scala.wasm', lua: 'tree-sitter-lua.wasm', r: 'tree-sitter-r.wasm', luau: 'tree-sitter-luau.wasm', objc: 'tree-sitter-objc.wasm', cfml: 'tree-sitter-cfml.wasm', cfscript: 'tree-sitter-cfscript.wasm', cfquery: 'tree-sitter-cfquery.wasm', cobol: 'tree-sitter-cobol.wasm', vbnet: 'tree-sitter-vbnet.wasm', erlang: 'tree-sitter-erlang.wasm', solidity: 'tree-sitter-solidity.wasm', terraform: 'tree-sitter-terraform.wasm', arkts: 'tree-sitter-arkts.wasm', nix: 'tree-sitter-nix.wasm', }; /** * File extension to Language mapping */ export const EXTENSION_MAP: Record = { '.ts': 'typescript', '.tsx': 'tsx', // ESM/CJS TypeScript module extensions — parsed as TS (no JSX). (#366) '.mts': 'typescript', '.cts': 'typescript', // ArkTS (HarmonyOS / OpenHarmony) — a TypeScript superset with declarative // UI (`@Component struct` + `build()`). Own grammar (a tree-sitter-typescript // -style fork); plain `.ts` in an ArkTS project stays TypeScript. (#648) '.ets': 'arkts', '.js': 'javascript', '.mjs': 'javascript', '.cjs': 'javascript', // SAP HANA XS Classic server-side JavaScript. (#556) '.xsjs': 'javascript', '.xsjslib': 'javascript', '.jsx': 'jsx', '.py': 'python', '.pyw': 'python', '.go': 'go', '.rs': 'rust', '.java': 'java', '.c': 'c', '.h': 'c', // Could also be C++, defaulting to C '.cpp': 'cpp', '.cc': 'cpp', '.cxx': 'cpp', '.hpp': 'cpp', '.hxx': 'cpp', '.cs': 'csharp', // ASP.NET Razor / Blazor markup — custom RazorExtractor (links @model/@inject/ // component tags to their C# types; markup isn't a tree-sitter grammar). '.cshtml': 'razor', '.razor': 'razor', '.php': 'php', // Drupal-specific PHP file extensions '.module': 'php', '.install': 'php', '.theme': 'php', '.inc': 'php', // YAML (used for Drupal routing files; no symbol extraction, file-level tracking only) '.yml': 'yaml', '.yaml': 'yaml', // Twig templates (file-level tracking only, no symbol extraction) '.twig': 'twig', '.rb': 'ruby', '.rake': 'ruby', '.swift': 'swift', '.kt': 'kotlin', '.kts': 'kotlin', '.dart': 'dart', '.liquid': 'liquid', '.svelte': 'svelte', '.vue': 'vue', '.astro': 'astro', '.r': 'r', '.pas': 'pascal', '.dpr': 'pascal', '.dpk': 'pascal', '.lpr': 'pascal', '.dfm': 'pascal', '.fmx': 'pascal', '.scala': 'scala', '.sc': 'scala', '.lua': 'lua', '.luau': 'luau', '.m': 'objc', '.mm': 'objc', '.sol': 'solidity', // CFML: .cfc/.cfm parse with the tag-aware `cfml` grammar (custom CfmlExtractor // dialect-switches to cfscript for bare-script content); .cfs is pure CFScript. '.cfc': 'cfml', '.cfm': 'cfml', '.cfs': 'cfscript', // Metal Shading Language ≈ C++14: the C++ grammar extracts its functions, // structs, and calls. MSL-specific `[[attribute]]` annotations are blanked // pre-parse for `.metal` files (see blankMetalAttributes in c-cpp.ts). (#1121) '.metal': 'cpp', // CUDA ≈ C++ plus execution-space specifiers (`__global__` …) and // `<<>>` kernel-launch syntax: the C++ grammar extracts its // functions/structs/classes/calls once blankCudaConstructs (pre-parse; gated // by these extensions OR by content for CUDA living in `.h`/`.hpp` headers — // see c-cpp.ts) blanks the CUDA-only tokens. (#387) '.cu': 'cpp', '.cuh': 'cpp', '.nix': 'nix', // XML: file-level tracking; the MyBatis extractor matches `` // shape and emits SQL-statement nodes (other XML returns empty). '.xml': 'xml', // COBOL: programs (.cbl/.cob) and copybooks (.cpy). Vendored grammar // (patched yutaro-sakamoto/tree-sitter-cobol) handles fixed-format column // rules, EXEC CICS/SQL blocks, and standalone copybook fragments. '.cbl': 'cobol', '.cob': 'cobol', '.cobol': 'cobol', '.cpy': 'cobol', // VB.NET: vendored grammar (patched govindbanura/tree-sitter-vbnet) — classes, // modules, interfaces, structures, properties, events, Handles clauses, LINQ. '.vb': 'vbnet', // Erlang: modules (.erl) and header files (.hrl). Vendored WhatsApp/ // tree-sitter-erlang grammar (the ELP grammar). '.erl': 'erlang', '.hrl': 'erlang', // escripts parse natively — the grammar has a first-class `shebang` node. // (`.app`/`.app.src` resource files route via isErlangAppFile below: their // last-dot extension is too generic for this map.) '.escript': 'erlang', // Spring config: `application.properties` / `application-*.properties`. Same // shape as the `.yml` variants — the YAML/properties extractor emits one node // per leaf key, and the Spring resolver links `@Value("${k}")` references. '.properties': 'properties', // Terraform / OpenTofu / HCL config — tree-sitter-terraform dialect of HCL. '.tf': 'terraform', '.tfvars': 'terraform', '.tofu': 'terraform', }; /** * Whether a file is one CodeGraph can parse, based purely on its extension. * This is the single source of truth for "should we index this file" — derived * from EXTENSION_MAP so parser support and indexing selection never drift. * * `overrides` is the project's validated custom extension → language map (from * `codegraph.json`); when present its extensions count as indexable in addition * to the built-ins. Omitting it is byte-identical to the zero-config behavior. */ export function isSourceFile(filePath: string, overrides?: Record): boolean { if (isPlayRoutesFile(filePath)) return true; // Play `conf/routes` is extensionless if (isShopifyLiquidJson(filePath)) return true; // Shopify OS 2.0 JSON templates / section groups if (isErlangAppFile(filePath)) return true; // OTP `.app`/`.app.src` resource files const dot = filePath.lastIndexOf('.'); if (dot < 0) return false; const ext = filePath.slice(dot).toLowerCase(); return ext in EXTENSION_MAP || (!!overrides && ext in overrides); } /** * Shopify OS 2.0 JSON template (`templates/*.json`) or section group * (`sections/*.json`) — these reference sections by `"type"`, so the Liquid * extractor links them. (config/ + locales/ JSON have no section refs.) */ export function isShopifyLiquidJson(filePath: string): boolean { // Allow nested template dirs (`templates/customers/login.json`), not just // top-level (`templates/product.json`). return /(^|\/)(templates|sections)\/.+\.json$/i.test(filePath); } /** * OTP application resource file: `.app.src` (checked into every rebar3/ * erlang.mk app) or its compiled `.app`. Erlang TERMS, not forms — the * grammar parses them as top-level expressions, and the Erlang extractor's * application-tuple handler turns `{mod, {Mod, _}}` and `{applications, […]}` * into entry-module and dependency edges. Routed by full suffix because the * last-dot extension (`.src`) is far too generic for EXTENSION_MAP. */ export function isErlangAppFile(filePath: string): boolean { return /\.app(?:\.src)?$/i.test(filePath); } /** * Play Framework routes file: the extensionless `conf/routes` (and included * `conf/*.routes`). No grammar — route extraction is done by the Play framework * resolver, so it's processed through the no-grammar (`yaml`-style) path. */ export function isPlayRoutesFile(filePath: string): boolean { return ( filePath === 'conf/routes' || filePath.endsWith('/conf/routes') || filePath.endsWith('.routes') ); } /** * Caches for loaded grammars and parsers */ const parserCache = new Map(); const languageCache = new Map(); const unavailableGrammarErrors = new Map(); let parserInitialized = false; /** * Initialize the tree-sitter WASM runtime. Must be called before loading grammars. * Does NOT load any grammar WASM files — use loadGrammarsForLanguages() for that. * Idempotent — safe to call multiple times. */ export async function initGrammars(): Promise { if (parserInitialized) return; await Parser.init(); parserInitialized = true; } /** * Grammars that ship their own vendored WASMs under `dist/extraction/wasm/` * (not in tree-sitter-wasms, or the tree-sitter-wasms build is too old). * Lua: tree-sitter-wasms ships an ABI-13 build that corrupts the shared WASM * heap under web-tree-sitter 0.25 (drops nested calls/imports on every file * after the first); we vendor the upstream ABI-15 wasm instead. C#: the * tree-sitter-wasms build (ABI 13) has no primary-constructor support and * parses `class Foo(...)` as an ERROR that swallows the whole class (#237); we * vendor the upstream ABI-15 tree-sitter-c-sharp 0.23.5 wasm, which parses * primary constructors natively. Terraform: tree-sitter-wasms does not ship * HCL/Terraform at all, so we vendor the prebuilt tree-sitter-terraform.wasm * from @tree-sitter-grammars/tree-sitter-hcl 1.2.0 (Apache-2.0) — * byte-identical to the npm package's artifact. ArkTS: tree-sitter-wasms * doesn't ship it either; we vendor the prebuilt tree-sitter-arkts.wasm from * the tree-sitter-arkts 0.2.0 npm package (harmony-contrib/tree-sitter-arkts, * MIT) — byte-identical to the npm tarball's artifact. It extends the * tree-sitter-javascript grammar the same way tree-sitter-typescript does, * adding `struct_declaration` and the `arkui_component_expression` build() * DSL. Nix: tree-sitter-wasms doesn't ship it; we vendor a wasm built from * nix-community/tree-sitter-nix @ 3d0173d (MIT) with tree-sitter-cli 0.25.10 * (`generate` + `build --wasm`, ABI 15 — upstream's checked-in parser.c is * still ABI 13; all 54 upstream corpus tests pass on the regenerated parser). * * TypeScript/TSX/JavaScript (+jsx, which shares the javascript grammar): the * tree-sitter-wasms builds are 2023-era (^0.20.x); we vendor wasm built from * the SAME grammar revisions the native extraction kernel compiles * (codegraph-kernel/Cargo.toml), so the kernel path and the wasm fallback * parse identically and per-language routing stays graph-neutral: * - tree-sitter/tree-sitter-typescript v0.23.2 (f975a62) → typescript + tsx * - tree-sitter/tree-sitter-javascript v0.25.0 (44c892e) → javascript + jsx * - tree-sitter/tree-sitter-java v0.23.5 (94703d5) → java * - tree-sitter/tree-sitter-python v0.23.6 (bffb65a) → python * - tree-sitter/tree-sitter-go v0.23.4 (3c3775f) → go * Built from each repo's CHECKED-IN parser.c (no `generate`) with * tree-sitter-cli 0.25.10 `build --wasm` — the same tables crates.io compiles * (parser.c sha-matched against the crates.io tarball). * The kernel-grammar-parity test asserts this alignment; bump the crate and * the vendored wasm together. */ const VENDORED_WASM_LANGS: ReadonlySet = new Set([ 'pascal', 'scala', 'lua', 'luau', 'csharp', 'r', 'cfml', 'cfscript', 'cfquery', 'cobol', 'vbnet', 'erlang', 'terraform', 'arkts', 'nix', 'typescript', 'tsx', 'javascript', 'jsx', 'java', 'python', 'go', // R7a (C/C++ kernel port prep): tree-sitter-c v0.24.2 (b780e47) + // tree-sitter-cpp v0.23.4 (f41e1a0), parser.c/scanner.c sha-matched against // the crates.io tarballs. `.metal`/`.cu` map to language 'cpp', so the // dialects ride the same (single, coherent) upgraded grammar. 'c', 'cpp', ]); /** Absolute path of a language's grammar WASM (vendored or tree-sitter-wasms). */ function resolveWasmPath(lang: GrammarLanguage): string { const wasmFile = WASM_GRAMMAR_FILES[lang]; return VENDORED_WASM_LANGS.has(lang) ? path.join(__dirname, 'wasm', wasmFile) : require.resolve(`tree-sitter-wasms/out/${wasmFile}`); } /** * Expand an index set's languages to the grammars actually needed to parse it. * SFC languages (svelte/vue/astro) have no grammar of their own — their * extractors delegate