Merge origin/main into delphi-support

Integrate main branch changes (WASM grammar architecture, centralized
resolution caches, SQLite adapter) with delphi-support branch. Pascal
grammar is now built as WASM and shipped in src/extraction/wasm/ for
consistency with the WASM-based grammar loading approach.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Olaf Monien
2026-02-17 20:23:54 +01:00
co-authored by Claude Opus 4.6
33 changed files with 1266 additions and 958 deletions
+70 -111
View File
@@ -1,91 +1,39 @@
/**
* Grammar Loading and Caching
*
* Uses lazy per-language loading so one missing native grammar does not
* break extraction for all other languages.
* Uses web-tree-sitter (WASM) for universal cross-platform support.
* All grammars are pre-loaded asynchronously via initGrammars(), then
* getParser() returns synchronously from cache.
*/
import Parser from 'tree-sitter';
import * as path from 'path';
import { Parser, Language as WasmLanguage } from 'web-tree-sitter';
import { Language } from '../types';
type GrammarLoader = () => unknown;
type GrammarLanguage = Exclude<Language, 'svelte' | 'liquid' | 'unknown'>;
/**
* Lazy grammar loaders — each language's native binding is only loaded
* on first use, so a failure in one grammar doesn't affect others.
* WASM filename map — maps each language to its .wasm grammar file
* in the tree-sitter-wasms package.
*/
const grammarLoaders: Record<GrammarLanguage, GrammarLoader> = {
typescript: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-typescript').typescript;
},
tsx: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-typescript').tsx;
},
javascript: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-javascript');
},
jsx: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-javascript');
},
python: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-python');
},
go: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-go');
},
rust: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-rust');
},
java: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-java');
},
c: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-c');
},
cpp: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-cpp');
},
csharp: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-c-sharp');
},
php: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-php').php;
},
ruby: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-ruby');
},
swift: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-swift');
},
kotlin: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-kotlin');
},
dart: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('@sengac/tree-sitter-dart');
},
pascal: () => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('tree-sitter-pascal');
},
// Note: tree-sitter-liquid has ABI compatibility issues with tree-sitter 0.22+
// Liquid extraction is handled separately via regex in tree-sitter.ts
const WASM_GRAMMAR_FILES: Record<GrammarLanguage, string> = {
typescript: 'tree-sitter-typescript.wasm',
tsx: 'tree-sitter-tsx.wasm',
javascript: 'tree-sitter-javascript.wasm',
jsx: 'tree-sitter-javascript.wasm',
python: 'tree-sitter-python.wasm',
go: 'tree-sitter-go.wasm',
rust: 'tree-sitter-rust.wasm',
java: 'tree-sitter-java.wasm',
c: 'tree-sitter-c.wasm',
cpp: 'tree-sitter-cpp.wasm',
csharp: 'tree-sitter-c_sharp.wasm',
php: 'tree-sitter-php.wasm',
ruby: 'tree-sitter-ruby.wasm',
swift: 'tree-sitter-swift.wasm',
kotlin: 'tree-sitter-kotlin.wasm',
dart: 'tree-sitter-dart.wasm',
pascal: 'tree-sitter-pascal.wasm',
};
/**
@@ -132,55 +80,65 @@ export const EXTENSION_MAP: Record<string, Language> = {
* Caches for loaded grammars and parsers
*/
const parserCache = new Map<Language, Parser>();
const grammarCache = new Map<Language, unknown | null>();
const languageCache = new Map<Language, WasmLanguage>();
const unavailableGrammarErrors = new Map<Language, string>();
let grammarsInitialized = false;
/**
* Load a grammar on demand, caching the result.
* Returns null if the grammar is not available on this platform.
* Initialize all WASM grammars. Must be called before any parsing.
* Idempotent — safe to call multiple times.
*/
function loadGrammar(language: Language): unknown | null {
if (grammarCache.has(language)) {
return grammarCache.get(language) ?? null;
}
export async function initGrammars(): Promise<void> {
if (grammarsInitialized) return;
const loader = grammarLoaders[language as GrammarLanguage];
if (!loader) {
grammarCache.set(language, null);
return null;
}
await Parser.init();
try {
const grammar = loader();
if (!grammar) {
throw new Error(`Grammar loader returned empty value for ${language}`);
}
grammarCache.set(language, grammar);
return grammar;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.warn(`[CodeGraph] Failed to load ${language} grammar — parsing will be unavailable: ${message}`);
unavailableGrammarErrors.set(language, message);
grammarCache.set(language, null);
return null;
}
// Load all grammars in parallel
const entries = Object.entries(WASM_GRAMMAR_FILES) as [GrammarLanguage, string][];
await Promise.allSettled(
entries.map(async ([lang, wasmFile]) => {
try {
// Pascal ships its own WASM (not in tree-sitter-wasms)
const wasmPath = lang === 'pascal'
? path.join(__dirname, 'wasm', wasmFile)
: require.resolve(`tree-sitter-wasms/out/${wasmFile}`);
const language = await WasmLanguage.load(wasmPath);
languageCache.set(lang, language);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.warn(`[CodeGraph] Failed to load ${lang} grammar — parsing will be unavailable: ${message}`);
unavailableGrammarErrors.set(lang, message);
}
})
);
grammarsInitialized = true;
}
/**
* Get a parser for the specified language
* Check if grammars have been initialized
*/
export function isGrammarsInitialized(): boolean {
return grammarsInitialized;
}
/**
* Get a parser for the specified language.
* Returns synchronously from pre-loaded cache.
*/
export function getParser(language: Language): Parser | null {
if (parserCache.has(language)) {
return parserCache.get(language)!;
}
const grammar = loadGrammar(language);
if (!grammar) {
const lang = languageCache.get(language);
if (!lang) {
return null;
}
const parser = new Parser();
parser.setLanguage(grammar as Parameters<typeof parser.setLanguage>[0]);
parser.setLanguage(lang);
parserCache.set(language, parser);
return parser;
}
@@ -200,15 +158,15 @@ export function isLanguageSupported(language: Language): boolean {
if (language === 'svelte') return true; // custom extractor (script block delegation)
if (language === 'liquid') return true; // custom regex extractor
if (language === 'unknown') return false;
return loadGrammar(language) !== null;
return languageCache.has(language);
}
/**
* Get all currently supported languages.
*/
export function getSupportedLanguages(): Language[] {
const available = (Object.keys(grammarLoaders) as GrammarLanguage[])
.filter((language) => loadGrammar(language) !== null);
const available = (Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[])
.filter((language) => languageCache.has(language));
return [...available, 'svelte', 'liquid'];
}
@@ -217,7 +175,8 @@ export function getSupportedLanguages(): Language[] {
*/
export function clearParserCache(): void {
parserCache.clear();
grammarCache.clear();
// Note: languageCache is NOT cleared — WASM languages persist.
// To fully re-init, set grammarsInitialized = false and call initGrammars() again.
unavailableGrammarErrors.clear();
}
+250 -71
View File
@@ -18,7 +18,7 @@ import {
} from '../types';
import { QueryBuilder } from '../db/queries';
import { extractFromSource } from './tree-sitter';
import { detectLanguage, isLanguageSupported } from './grammars';
import { detectLanguage, isLanguageSupported, initGrammars } from './grammars';
import { logDebug, logWarn } from '../errors';
import { captureException } from '../sentry';
import { validatePathWithinRoot, normalizePath } from '../utils';
@@ -63,6 +63,7 @@ export interface SyncResult {
filesRemoved: number;
nodesUpdated: number;
durationMs: number;
changedFilePaths?: string[];
}
/**
@@ -105,27 +106,80 @@ export function shouldIncludeFile(
}
/**
* Get directories ignored by .gitignore using git ls-files.
* Returns a Set of normalized relative directory paths (forward slashes, no trailing slash).
* Gracefully returns empty Set on any failure.
* Get all files visible to git (tracked + untracked but not ignored).
* Respects .gitignore at all levels (root, subdirectories).
* Returns null on failure (non-git project) so callers can fall back.
*/
function getGitIgnoredDirectories(rootDir: string): Set<string> {
function getGitVisibleFiles(rootDir: string): Set<string> | null {
try {
// -c = cached (tracked), -o = others (untracked), --exclude-standard = respect .gitignore
const output = execFileSync(
'git',
['ls-files', '-co', '--exclude-standard'],
{ cwd: rootDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] }
);
const files = new Set<string>();
for (const line of output.split('\n')) {
const trimmed = line.trim();
if (trimmed) {
files.add(normalizePath(trimmed));
}
}
return files;
} catch {
return null;
}
}
/**
* Result of git-based change detection.
* Returns null when git is unavailable (non-git project or command failure),
* signaling the caller to fall back to full filesystem scan.
*/
interface GitChanges {
modified: string[]; // M, MM, AM — files to re-hash + re-index
added: string[]; // ?? — new untracked files to index
deleted: string[]; // D — files to remove from DB
}
/**
* Use `git status` to detect changed files instead of scanning every file.
* Returns null on failure so callers fall back to full scan.
*/
function getGitChangedFiles(rootDir: string, config: CodeGraphConfig): GitChanges | null {
try {
const output = execFileSync(
'git',
['ls-files', '-oi', '--exclude-standard', '--directory'],
['status', '--porcelain', '--no-renames'],
{ cwd: rootDir, encoding: 'utf-8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'] }
);
const dirs = new Set<string>();
const modified: string[] = [];
const added: string[] = [];
const deleted: string[] = [];
for (const line of output.split('\n')) {
const trimmed = line.trim();
if (trimmed.endsWith('/')) {
dirs.add(normalizePath(trimmed.slice(0, -1)));
if (line.length < 4) continue; // Minimum: "XY file"
const statusCode = line.substring(0, 2);
const filePath = normalizePath(line.substring(3));
// Skip files that don't match include/exclude config
if (!shouldIncludeFile(filePath, config)) continue;
if (statusCode === '??') {
added.push(filePath);
} else if (statusCode.includes('D')) {
deleted.push(filePath);
} else {
// M, MM, AM, A (staged), etc. — treat as modified
modified.push(filePath);
}
}
return dirs;
return { modified, added, deleted };
} catch {
return new Set<string>();
return null;
}
}
@@ -135,21 +189,49 @@ function getGitIgnoredDirectories(rootDir: string): Set<string> {
const CODEGRAPH_IGNORE_MARKER = '.codegraphignore';
/**
* Recursively scan directory for source files
* Recursively scan directory for source files.
*
* In git repos, uses `git ls-files` to get the file list (inherently
* respects .gitignore at all levels), then filters by config include patterns.
* Falls back to filesystem walk for non-git projects.
*/
export function scanDirectory(
rootDir: string,
config: CodeGraphConfig,
onProgress?: (current: number, file: string) => void
): string[] {
// Fast path: use git to get all visible files (respects .gitignore everywhere)
const gitFiles = getGitVisibleFiles(rootDir);
if (gitFiles) {
const files: string[] = [];
let count = 0;
for (const filePath of gitFiles) {
if (shouldIncludeFile(filePath, config)) {
files.push(filePath);
count++;
onProgress?.(count, filePath);
}
}
return files;
}
// Fallback: walk filesystem for non-git projects
return scanDirectoryWalk(rootDir, config, onProgress);
}
/**
* Filesystem walk fallback for non-git projects.
*/
function scanDirectoryWalk(
rootDir: string,
config: CodeGraphConfig,
onProgress?: (current: number, file: string) => void
): string[] {
const files: string[] = [];
let count = 0;
// Track visited real paths to detect symlink cycles
const visitedDirs = new Set<string>();
const gitIgnoredDirs = getGitIgnoredDirectories(rootDir);
function walk(dir: string): void {
// Resolve real path to detect symlink cycles
let realDir: string;
try {
realDir = fs.realpathSync(dir);
@@ -164,7 +246,7 @@ export function scanDirectory(
}
visitedDirs.add(realDir);
// Check for .codegraphignore marker file - skip entire directory tree if present
// Check for .codegraphignore marker file
const ignoreMarker = path.join(dir, CODEGRAPH_IGNORE_MARKER);
if (fs.existsSync(ignoreMarker)) {
logDebug('Skipping directory due to .codegraphignore marker', { dir });
@@ -184,17 +266,11 @@ export function scanDirectory(
const fullPath = path.join(dir, entry.name);
const relativePath = normalizePath(path.relative(rootDir, fullPath));
// Follow symlinked directories, but skip symlinked files to non-project targets
if (entry.isSymbolicLink()) {
try {
const realTarget = fs.realpathSync(fullPath);
const stat = fs.statSync(realTarget);
if (stat.isDirectory()) {
// Check gitignore first (fast O(1) lookup)
if (gitIgnoredDirs.has(relativePath)) {
continue;
}
// Check exclusion, then recurse (cycle detection handles the rest)
const dirPattern = relativePath + '/';
let excluded = false;
for (const pattern of config.exclude) {
@@ -210,9 +286,7 @@ export function scanDirectory(
if (shouldIncludeFile(relativePath, config)) {
files.push(relativePath);
count++;
if (onProgress) {
onProgress(count, relativePath);
}
onProgress?.(count, relativePath);
}
}
} catch {
@@ -222,11 +296,6 @@ export function scanDirectory(
}
if (entry.isDirectory()) {
// Check gitignore first (fast O(1) lookup)
if (gitIgnoredDirs.has(relativePath)) {
continue;
}
// Check if directory should be excluded
const dirPattern = relativePath + '/';
let excluded = false;
for (const pattern of config.exclude) {
@@ -242,9 +311,7 @@ export function scanDirectory(
if (shouldIncludeFile(relativePath, config)) {
files.push(relativePath);
count++;
if (onProgress) {
onProgress(count, relativePath);
}
onProgress?.(count, relativePath);
}
}
}
@@ -275,6 +342,7 @@ export class ExtractionOrchestrator {
onProgress?: (progress: IndexProgress) => void,
signal?: AbortSignal
): Promise<IndexResult> {
await initGrammars();
const startTime = Date.now();
const errors: ExtractionError[] = [];
let filesIndexed = 0;
@@ -611,62 +679,118 @@ export class ExtractionOrchestrator {
}
/**
* Sync with current file state
* Sync with current file state.
* Uses git status as a fast path when available, falling back to full scan.
*/
async sync(onProgress?: (progress: IndexProgress) => void): Promise<SyncResult> {
await initGrammars();
const startTime = Date.now();
let filesChecked = 0;
let filesAdded = 0;
let filesModified = 0;
let filesRemoved = 0;
let nodesUpdated = 0;
const changedFilePaths: string[] = [];
// Get current files on disk
onProgress?.({
phase: 'scanning',
current: 0,
total: 0,
});
const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
filesChecked = currentFiles.size;
// Get tracked files from database
const trackedFiles = this.queries.getAllFiles();
// Find files to remove (in DB but not on disk)
for (const tracked of trackedFiles) {
if (!currentFiles.has(tracked.path)) {
this.queries.deleteFile(tracked.path);
filesRemoved++;
}
}
// Find files to add or update
const filesToIndex: string[] = [];
const gitChanges = getGitChangedFiles(this.rootDir, this.config);
for (const filePath of currentFiles) {
const fullPath = path.join(this.rootDir, filePath);
let content: string;
try {
content = fs.readFileSync(fullPath, 'utf-8');
} catch (error) {
captureException(error, { operation: 'sync-read-file', filePath });
logDebug('Skipping unreadable file during sync', { filePath, error: String(error) });
continue;
if (gitChanges) {
// === Git fast path ===
// Only inspect the files git reports as changed instead of scanning everything.
filesChecked = gitChanges.modified.length + gitChanges.added.length + gitChanges.deleted.length;
// Handle deleted files
for (const filePath of gitChanges.deleted) {
const tracked = this.queries.getFileByPath(filePath);
if (tracked) {
this.queries.deleteFile(filePath);
filesRemoved++;
}
}
const contentHash = hashContent(content);
const tracked = trackedFiles.find((f) => f.path === filePath);
// Handle modified files — read + hash only these files
for (const filePath of gitChanges.modified) {
const fullPath = path.join(this.rootDir, filePath);
let content: string;
try {
content = fs.readFileSync(fullPath, 'utf-8');
} catch (error) {
captureException(error, { operation: 'sync-read-file', filePath });
logDebug('Skipping unreadable file during sync', { filePath, error: String(error) });
continue;
}
if (!tracked) {
// New file
const contentHash = hashContent(content);
const tracked = this.queries.getFileByPath(filePath);
if (!tracked) {
filesToIndex.push(filePath);
changedFilePaths.push(filePath);
filesAdded++;
} else if (tracked.contentHash !== contentHash) {
filesToIndex.push(filePath);
changedFilePaths.push(filePath);
filesModified++;
}
}
// Handle added (untracked) files
for (const filePath of gitChanges.added) {
filesToIndex.push(filePath);
changedFilePaths.push(filePath);
filesAdded++;
} else if (tracked.contentHash !== contentHash) {
// Modified file
filesToIndex.push(filePath);
filesModified++;
}
} else {
// === Fallback: full scan (non-git project or git failure) ===
const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
filesChecked = currentFiles.size;
// Build Map for O(1) lookups instead of .find() per file
const trackedFiles = this.queries.getAllFiles();
const trackedMap = new Map<string, FileRecord>();
for (const f of trackedFiles) {
trackedMap.set(f.path, f);
}
// Find files to remove (in DB but not on disk)
for (const tracked of trackedFiles) {
if (!currentFiles.has(tracked.path)) {
this.queries.deleteFile(tracked.path);
filesRemoved++;
}
}
// Find files to add or update
for (const filePath of currentFiles) {
const fullPath = path.join(this.rootDir, filePath);
let content: string;
try {
content = fs.readFileSync(fullPath, 'utf-8');
} catch (error) {
captureException(error, { operation: 'sync-read-file', filePath });
logDebug('Skipping unreadable file during sync', { filePath, error: String(error) });
continue;
}
const contentHash = hashContent(content);
const tracked = trackedMap.get(filePath);
if (!tracked) {
filesToIndex.push(filePath);
changedFilePaths.push(filePath);
filesAdded++;
} else if (tracked.contentHash !== contentHash) {
filesToIndex.push(filePath);
changedFilePaths.push(filePath);
filesModified++;
}
}
}
@@ -692,16 +816,71 @@ export class ExtractionOrchestrator {
filesRemoved,
nodesUpdated,
durationMs: Date.now() - startTime,
changedFilePaths: changedFilePaths.length > 0 ? changedFilePaths : undefined,
};
}
/**
* Get files that have changed since last index
* Get files that have changed since last index.
* Uses git status as a fast path when available, falling back to full scan.
*/
getChangedFiles(): { added: string[]; modified: string[]; removed: string[] } {
const gitChanges = getGitChangedFiles(this.rootDir, this.config);
if (gitChanges) {
// === Git fast path ===
const added: string[] = [];
const modified: string[] = [];
const removed: string[] = [];
// Deleted files — only report if tracked in DB
for (const filePath of gitChanges.deleted) {
const tracked = this.queries.getFileByPath(filePath);
if (tracked) {
removed.push(filePath);
}
}
// Modified files — read + hash only these, compare with DB
for (const filePath of gitChanges.modified) {
const fullPath = path.join(this.rootDir, filePath);
let content: string;
try {
content = fs.readFileSync(fullPath, 'utf-8');
} catch (error) {
captureException(error, { operation: 'detect-changes-read-file', filePath });
logDebug('Skipping unreadable file while detecting changes', { filePath, error: String(error) });
continue;
}
const contentHash = hashContent(content);
const tracked = this.queries.getFileByPath(filePath);
if (!tracked) {
added.push(filePath);
} else if (tracked.contentHash !== contentHash) {
modified.push(filePath);
}
}
// Added (untracked) files
for (const filePath of gitChanges.added) {
added.push(filePath);
}
return { added, modified, removed };
}
// === Fallback: full scan (non-git project or git failure) ===
const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
const trackedFiles = this.queries.getAllFiles();
// Build Map for O(1) lookups
const trackedMap = new Map<string, FileRecord>();
for (const f of trackedFiles) {
trackedMap.set(f.path, f);
}
const added: string[] = [];
const modified: string[] = [];
const removed: string[] = [];
@@ -726,7 +905,7 @@ export class ExtractionOrchestrator {
}
const contentHash = hashContent(content);
const tracked = trackedFiles.find((f) => f.path === filePath);
const tracked = trackedMap.get(filePath);
if (!tracked) {
added.push(filePath);
@@ -741,4 +920,4 @@ export class ExtractionOrchestrator {
// Re-export useful types and functions
export { extractFromSource } from './tree-sitter';
export { detectLanguage, isLanguageSupported, getSupportedLanguages } from './grammars';
export { detectLanguage, isLanguageSupported, getSupportedLanguages, initGrammars } from './grammars';
+13 -4
View File
@@ -4,7 +4,7 @@
* Handles parsing source code and extracting structural information.
*/
import { SyntaxNode, Tree } from 'tree-sitter';
import { Node as SyntaxNode, Tree } from 'web-tree-sitter';
import * as crypto from 'crypto';
import * as path from 'path';
import {
@@ -934,7 +934,10 @@ export class TreeSitterExtractor {
}
try {
this.tree = parser.parse(this.source);
this.tree = parser.parse(this.source) ?? null;
if (!this.tree) {
throw new Error('Parser returned null tree');
}
// Create file node representing the source file
const fileNode: Node = {
@@ -1775,9 +1778,15 @@ export class TreeSitterExtractor {
if (namespacePrefix && useGroup) {
// Grouped import - create one import per item
const prefix = getNodeText(namespacePrefix, this.source);
const useClauses = useGroup.namedChildren.filter((c: SyntaxNode) => c.type === 'namespace_use_clause');
const useClauses = useGroup.namedChildren.filter((c: SyntaxNode) =>
c.type === 'namespace_use_group_clause' || c.type === 'namespace_use_clause'
);
for (const clause of useClauses) {
const name = clause.namedChildren.find((c: SyntaxNode) => c.type === 'name');
// WASM grammar wraps names in namespace_name; native uses name directly
const nsName = clause.namedChildren.find((c: SyntaxNode) => c.type === 'namespace_name');
const name = nsName
? nsName.namedChildren.find((c: SyntaxNode) => c.type === 'name')
: clause.namedChildren.find((c: SyntaxNode) => c.type === 'name');
if (name) {
const fullPath = `${prefix}\\${getNodeText(name, this.source)}`;
this.createNode('import', fullPath, node, {
Binary file not shown.