Merge origin/main into delphi-support
Integrate main branch changes (WASM grammar architecture, centralized resolution caches, SQLite adapter) with delphi-support branch. Pascal grammar is now built as WASM and shipped in src/extraction/wasm/ for consistency with the WASM-based grammar loading approach. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+70
-111
@@ -1,91 +1,39 @@
|
||||
/**
|
||||
* Grammar Loading and Caching
|
||||
*
|
||||
* Uses lazy per-language loading so one missing native grammar does not
|
||||
* break extraction for all other languages.
|
||||
* Uses web-tree-sitter (WASM) for universal cross-platform support.
|
||||
* All grammars are pre-loaded asynchronously via initGrammars(), then
|
||||
* getParser() returns synchronously from cache.
|
||||
*/
|
||||
|
||||
import Parser from 'tree-sitter';
|
||||
import * as path from 'path';
|
||||
import { Parser, Language as WasmLanguage } from 'web-tree-sitter';
|
||||
import { Language } from '../types';
|
||||
|
||||
type GrammarLoader = () => unknown;
|
||||
type GrammarLanguage = Exclude<Language, 'svelte' | 'liquid' | 'unknown'>;
|
||||
|
||||
/**
|
||||
* Lazy grammar loaders — each language's native binding is only loaded
|
||||
* on first use, so a failure in one grammar doesn't affect others.
|
||||
* WASM filename map — maps each language to its .wasm grammar file
|
||||
* in the tree-sitter-wasms package.
|
||||
*/
|
||||
const grammarLoaders: Record<GrammarLanguage, GrammarLoader> = {
|
||||
typescript: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-typescript').typescript;
|
||||
},
|
||||
tsx: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-typescript').tsx;
|
||||
},
|
||||
javascript: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-javascript');
|
||||
},
|
||||
jsx: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-javascript');
|
||||
},
|
||||
python: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-python');
|
||||
},
|
||||
go: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-go');
|
||||
},
|
||||
rust: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-rust');
|
||||
},
|
||||
java: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-java');
|
||||
},
|
||||
c: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-c');
|
||||
},
|
||||
cpp: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-cpp');
|
||||
},
|
||||
csharp: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-c-sharp');
|
||||
},
|
||||
php: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-php').php;
|
||||
},
|
||||
ruby: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-ruby');
|
||||
},
|
||||
swift: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-swift');
|
||||
},
|
||||
kotlin: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-kotlin');
|
||||
},
|
||||
dart: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('@sengac/tree-sitter-dart');
|
||||
},
|
||||
pascal: () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('tree-sitter-pascal');
|
||||
},
|
||||
// Note: tree-sitter-liquid has ABI compatibility issues with tree-sitter 0.22+
|
||||
// Liquid extraction is handled separately via regex in tree-sitter.ts
|
||||
const WASM_GRAMMAR_FILES: Record<GrammarLanguage, string> = {
|
||||
typescript: 'tree-sitter-typescript.wasm',
|
||||
tsx: 'tree-sitter-tsx.wasm',
|
||||
javascript: 'tree-sitter-javascript.wasm',
|
||||
jsx: 'tree-sitter-javascript.wasm',
|
||||
python: 'tree-sitter-python.wasm',
|
||||
go: 'tree-sitter-go.wasm',
|
||||
rust: 'tree-sitter-rust.wasm',
|
||||
java: 'tree-sitter-java.wasm',
|
||||
c: 'tree-sitter-c.wasm',
|
||||
cpp: 'tree-sitter-cpp.wasm',
|
||||
csharp: 'tree-sitter-c_sharp.wasm',
|
||||
php: 'tree-sitter-php.wasm',
|
||||
ruby: 'tree-sitter-ruby.wasm',
|
||||
swift: 'tree-sitter-swift.wasm',
|
||||
kotlin: 'tree-sitter-kotlin.wasm',
|
||||
dart: 'tree-sitter-dart.wasm',
|
||||
pascal: 'tree-sitter-pascal.wasm',
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -132,55 +80,65 @@ export const EXTENSION_MAP: Record<string, Language> = {
|
||||
* Caches for loaded grammars and parsers
|
||||
*/
|
||||
const parserCache = new Map<Language, Parser>();
|
||||
const grammarCache = new Map<Language, unknown | null>();
|
||||
const languageCache = new Map<Language, WasmLanguage>();
|
||||
const unavailableGrammarErrors = new Map<Language, string>();
|
||||
|
||||
let grammarsInitialized = false;
|
||||
|
||||
/**
|
||||
* Load a grammar on demand, caching the result.
|
||||
* Returns null if the grammar is not available on this platform.
|
||||
* Initialize all WASM grammars. Must be called before any parsing.
|
||||
* Idempotent — safe to call multiple times.
|
||||
*/
|
||||
function loadGrammar(language: Language): unknown | null {
|
||||
if (grammarCache.has(language)) {
|
||||
return grammarCache.get(language) ?? null;
|
||||
}
|
||||
export async function initGrammars(): Promise<void> {
|
||||
if (grammarsInitialized) return;
|
||||
|
||||
const loader = grammarLoaders[language as GrammarLanguage];
|
||||
if (!loader) {
|
||||
grammarCache.set(language, null);
|
||||
return null;
|
||||
}
|
||||
await Parser.init();
|
||||
|
||||
try {
|
||||
const grammar = loader();
|
||||
if (!grammar) {
|
||||
throw new Error(`Grammar loader returned empty value for ${language}`);
|
||||
}
|
||||
grammarCache.set(language, grammar);
|
||||
return grammar;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.warn(`[CodeGraph] Failed to load ${language} grammar — parsing will be unavailable: ${message}`);
|
||||
unavailableGrammarErrors.set(language, message);
|
||||
grammarCache.set(language, null);
|
||||
return null;
|
||||
}
|
||||
// Load all grammars in parallel
|
||||
const entries = Object.entries(WASM_GRAMMAR_FILES) as [GrammarLanguage, string][];
|
||||
await Promise.allSettled(
|
||||
entries.map(async ([lang, wasmFile]) => {
|
||||
try {
|
||||
// Pascal ships its own WASM (not in tree-sitter-wasms)
|
||||
const wasmPath = lang === 'pascal'
|
||||
? path.join(__dirname, 'wasm', wasmFile)
|
||||
: require.resolve(`tree-sitter-wasms/out/${wasmFile}`);
|
||||
const language = await WasmLanguage.load(wasmPath);
|
||||
languageCache.set(lang, language);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.warn(`[CodeGraph] Failed to load ${lang} grammar — parsing will be unavailable: ${message}`);
|
||||
unavailableGrammarErrors.set(lang, message);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
grammarsInitialized = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a parser for the specified language
|
||||
* Check if grammars have been initialized
|
||||
*/
|
||||
export function isGrammarsInitialized(): boolean {
|
||||
return grammarsInitialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a parser for the specified language.
|
||||
* Returns synchronously from pre-loaded cache.
|
||||
*/
|
||||
export function getParser(language: Language): Parser | null {
|
||||
if (parserCache.has(language)) {
|
||||
return parserCache.get(language)!;
|
||||
}
|
||||
|
||||
const grammar = loadGrammar(language);
|
||||
if (!grammar) {
|
||||
const lang = languageCache.get(language);
|
||||
if (!lang) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(grammar as Parameters<typeof parser.setLanguage>[0]);
|
||||
parser.setLanguage(lang);
|
||||
parserCache.set(language, parser);
|
||||
return parser;
|
||||
}
|
||||
@@ -200,15 +158,15 @@ export function isLanguageSupported(language: Language): boolean {
|
||||
if (language === 'svelte') return true; // custom extractor (script block delegation)
|
||||
if (language === 'liquid') return true; // custom regex extractor
|
||||
if (language === 'unknown') return false;
|
||||
return loadGrammar(language) !== null;
|
||||
return languageCache.has(language);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all currently supported languages.
|
||||
*/
|
||||
export function getSupportedLanguages(): Language[] {
|
||||
const available = (Object.keys(grammarLoaders) as GrammarLanguage[])
|
||||
.filter((language) => loadGrammar(language) !== null);
|
||||
const available = (Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[])
|
||||
.filter((language) => languageCache.has(language));
|
||||
return [...available, 'svelte', 'liquid'];
|
||||
}
|
||||
|
||||
@@ -217,7 +175,8 @@ export function getSupportedLanguages(): Language[] {
|
||||
*/
|
||||
export function clearParserCache(): void {
|
||||
parserCache.clear();
|
||||
grammarCache.clear();
|
||||
// Note: languageCache is NOT cleared — WASM languages persist.
|
||||
// To fully re-init, set grammarsInitialized = false and call initGrammars() again.
|
||||
unavailableGrammarErrors.clear();
|
||||
}
|
||||
|
||||
|
||||
+250
-71
@@ -18,7 +18,7 @@ import {
|
||||
} from '../types';
|
||||
import { QueryBuilder } from '../db/queries';
|
||||
import { extractFromSource } from './tree-sitter';
|
||||
import { detectLanguage, isLanguageSupported } from './grammars';
|
||||
import { detectLanguage, isLanguageSupported, initGrammars } from './grammars';
|
||||
import { logDebug, logWarn } from '../errors';
|
||||
import { captureException } from '../sentry';
|
||||
import { validatePathWithinRoot, normalizePath } from '../utils';
|
||||
@@ -63,6 +63,7 @@ export interface SyncResult {
|
||||
filesRemoved: number;
|
||||
nodesUpdated: number;
|
||||
durationMs: number;
|
||||
changedFilePaths?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -105,27 +106,80 @@ export function shouldIncludeFile(
|
||||
}
|
||||
|
||||
/**
|
||||
* Get directories ignored by .gitignore using git ls-files.
|
||||
* Returns a Set of normalized relative directory paths (forward slashes, no trailing slash).
|
||||
* Gracefully returns empty Set on any failure.
|
||||
* Get all files visible to git (tracked + untracked but not ignored).
|
||||
* Respects .gitignore at all levels (root, subdirectories).
|
||||
* Returns null on failure (non-git project) so callers can fall back.
|
||||
*/
|
||||
function getGitIgnoredDirectories(rootDir: string): Set<string> {
|
||||
function getGitVisibleFiles(rootDir: string): Set<string> | null {
|
||||
try {
|
||||
// -c = cached (tracked), -o = others (untracked), --exclude-standard = respect .gitignore
|
||||
const output = execFileSync(
|
||||
'git',
|
||||
['ls-files', '-co', '--exclude-standard'],
|
||||
{ cwd: rootDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] }
|
||||
);
|
||||
const files = new Set<string>();
|
||||
for (const line of output.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed) {
|
||||
files.add(normalizePath(trimmed));
|
||||
}
|
||||
}
|
||||
return files;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of git-based change detection.
|
||||
* Returns null when git is unavailable (non-git project or command failure),
|
||||
* signaling the caller to fall back to full filesystem scan.
|
||||
*/
|
||||
interface GitChanges {
|
||||
modified: string[]; // M, MM, AM — files to re-hash + re-index
|
||||
added: string[]; // ?? — new untracked files to index
|
||||
deleted: string[]; // D — files to remove from DB
|
||||
}
|
||||
|
||||
/**
|
||||
* Use `git status` to detect changed files instead of scanning every file.
|
||||
* Returns null on failure so callers fall back to full scan.
|
||||
*/
|
||||
function getGitChangedFiles(rootDir: string, config: CodeGraphConfig): GitChanges | null {
|
||||
try {
|
||||
const output = execFileSync(
|
||||
'git',
|
||||
['ls-files', '-oi', '--exclude-standard', '--directory'],
|
||||
['status', '--porcelain', '--no-renames'],
|
||||
{ cwd: rootDir, encoding: 'utf-8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'] }
|
||||
);
|
||||
const dirs = new Set<string>();
|
||||
|
||||
const modified: string[] = [];
|
||||
const added: string[] = [];
|
||||
const deleted: string[] = [];
|
||||
|
||||
for (const line of output.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.endsWith('/')) {
|
||||
dirs.add(normalizePath(trimmed.slice(0, -1)));
|
||||
if (line.length < 4) continue; // Minimum: "XY file"
|
||||
|
||||
const statusCode = line.substring(0, 2);
|
||||
const filePath = normalizePath(line.substring(3));
|
||||
|
||||
// Skip files that don't match include/exclude config
|
||||
if (!shouldIncludeFile(filePath, config)) continue;
|
||||
|
||||
if (statusCode === '??') {
|
||||
added.push(filePath);
|
||||
} else if (statusCode.includes('D')) {
|
||||
deleted.push(filePath);
|
||||
} else {
|
||||
// M, MM, AM, A (staged), etc. — treat as modified
|
||||
modified.push(filePath);
|
||||
}
|
||||
}
|
||||
return dirs;
|
||||
|
||||
return { modified, added, deleted };
|
||||
} catch {
|
||||
return new Set<string>();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,21 +189,49 @@ function getGitIgnoredDirectories(rootDir: string): Set<string> {
|
||||
const CODEGRAPH_IGNORE_MARKER = '.codegraphignore';
|
||||
|
||||
/**
|
||||
* Recursively scan directory for source files
|
||||
* Recursively scan directory for source files.
|
||||
*
|
||||
* In git repos, uses `git ls-files` to get the file list (inherently
|
||||
* respects .gitignore at all levels), then filters by config include patterns.
|
||||
* Falls back to filesystem walk for non-git projects.
|
||||
*/
|
||||
export function scanDirectory(
|
||||
rootDir: string,
|
||||
config: CodeGraphConfig,
|
||||
onProgress?: (current: number, file: string) => void
|
||||
): string[] {
|
||||
// Fast path: use git to get all visible files (respects .gitignore everywhere)
|
||||
const gitFiles = getGitVisibleFiles(rootDir);
|
||||
if (gitFiles) {
|
||||
const files: string[] = [];
|
||||
let count = 0;
|
||||
for (const filePath of gitFiles) {
|
||||
if (shouldIncludeFile(filePath, config)) {
|
||||
files.push(filePath);
|
||||
count++;
|
||||
onProgress?.(count, filePath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
// Fallback: walk filesystem for non-git projects
|
||||
return scanDirectoryWalk(rootDir, config, onProgress);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filesystem walk fallback for non-git projects.
|
||||
*/
|
||||
function scanDirectoryWalk(
|
||||
rootDir: string,
|
||||
config: CodeGraphConfig,
|
||||
onProgress?: (current: number, file: string) => void
|
||||
): string[] {
|
||||
const files: string[] = [];
|
||||
let count = 0;
|
||||
// Track visited real paths to detect symlink cycles
|
||||
const visitedDirs = new Set<string>();
|
||||
const gitIgnoredDirs = getGitIgnoredDirectories(rootDir);
|
||||
|
||||
function walk(dir: string): void {
|
||||
// Resolve real path to detect symlink cycles
|
||||
let realDir: string;
|
||||
try {
|
||||
realDir = fs.realpathSync(dir);
|
||||
@@ -164,7 +246,7 @@ export function scanDirectory(
|
||||
}
|
||||
visitedDirs.add(realDir);
|
||||
|
||||
// Check for .codegraphignore marker file - skip entire directory tree if present
|
||||
// Check for .codegraphignore marker file
|
||||
const ignoreMarker = path.join(dir, CODEGRAPH_IGNORE_MARKER);
|
||||
if (fs.existsSync(ignoreMarker)) {
|
||||
logDebug('Skipping directory due to .codegraphignore marker', { dir });
|
||||
@@ -184,17 +266,11 @@ export function scanDirectory(
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
const relativePath = normalizePath(path.relative(rootDir, fullPath));
|
||||
|
||||
// Follow symlinked directories, but skip symlinked files to non-project targets
|
||||
if (entry.isSymbolicLink()) {
|
||||
try {
|
||||
const realTarget = fs.realpathSync(fullPath);
|
||||
const stat = fs.statSync(realTarget);
|
||||
if (stat.isDirectory()) {
|
||||
// Check gitignore first (fast O(1) lookup)
|
||||
if (gitIgnoredDirs.has(relativePath)) {
|
||||
continue;
|
||||
}
|
||||
// Check exclusion, then recurse (cycle detection handles the rest)
|
||||
const dirPattern = relativePath + '/';
|
||||
let excluded = false;
|
||||
for (const pattern of config.exclude) {
|
||||
@@ -210,9 +286,7 @@ export function scanDirectory(
|
||||
if (shouldIncludeFile(relativePath, config)) {
|
||||
files.push(relativePath);
|
||||
count++;
|
||||
if (onProgress) {
|
||||
onProgress(count, relativePath);
|
||||
}
|
||||
onProgress?.(count, relativePath);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
@@ -222,11 +296,6 @@ export function scanDirectory(
|
||||
}
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
// Check gitignore first (fast O(1) lookup)
|
||||
if (gitIgnoredDirs.has(relativePath)) {
|
||||
continue;
|
||||
}
|
||||
// Check if directory should be excluded
|
||||
const dirPattern = relativePath + '/';
|
||||
let excluded = false;
|
||||
for (const pattern of config.exclude) {
|
||||
@@ -242,9 +311,7 @@ export function scanDirectory(
|
||||
if (shouldIncludeFile(relativePath, config)) {
|
||||
files.push(relativePath);
|
||||
count++;
|
||||
if (onProgress) {
|
||||
onProgress(count, relativePath);
|
||||
}
|
||||
onProgress?.(count, relativePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -275,6 +342,7 @@ export class ExtractionOrchestrator {
|
||||
onProgress?: (progress: IndexProgress) => void,
|
||||
signal?: AbortSignal
|
||||
): Promise<IndexResult> {
|
||||
await initGrammars();
|
||||
const startTime = Date.now();
|
||||
const errors: ExtractionError[] = [];
|
||||
let filesIndexed = 0;
|
||||
@@ -611,62 +679,118 @@ export class ExtractionOrchestrator {
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync with current file state
|
||||
* Sync with current file state.
|
||||
* Uses git status as a fast path when available, falling back to full scan.
|
||||
*/
|
||||
async sync(onProgress?: (progress: IndexProgress) => void): Promise<SyncResult> {
|
||||
await initGrammars();
|
||||
const startTime = Date.now();
|
||||
let filesChecked = 0;
|
||||
let filesAdded = 0;
|
||||
let filesModified = 0;
|
||||
let filesRemoved = 0;
|
||||
let nodesUpdated = 0;
|
||||
const changedFilePaths: string[] = [];
|
||||
|
||||
// Get current files on disk
|
||||
onProgress?.({
|
||||
phase: 'scanning',
|
||||
current: 0,
|
||||
total: 0,
|
||||
});
|
||||
|
||||
const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
|
||||
filesChecked = currentFiles.size;
|
||||
|
||||
// Get tracked files from database
|
||||
const trackedFiles = this.queries.getAllFiles();
|
||||
|
||||
// Find files to remove (in DB but not on disk)
|
||||
for (const tracked of trackedFiles) {
|
||||
if (!currentFiles.has(tracked.path)) {
|
||||
this.queries.deleteFile(tracked.path);
|
||||
filesRemoved++;
|
||||
}
|
||||
}
|
||||
|
||||
// Find files to add or update
|
||||
const filesToIndex: string[] = [];
|
||||
const gitChanges = getGitChangedFiles(this.rootDir, this.config);
|
||||
|
||||
for (const filePath of currentFiles) {
|
||||
const fullPath = path.join(this.rootDir, filePath);
|
||||
let content: string;
|
||||
try {
|
||||
content = fs.readFileSync(fullPath, 'utf-8');
|
||||
} catch (error) {
|
||||
captureException(error, { operation: 'sync-read-file', filePath });
|
||||
logDebug('Skipping unreadable file during sync', { filePath, error: String(error) });
|
||||
continue;
|
||||
if (gitChanges) {
|
||||
// === Git fast path ===
|
||||
// Only inspect the files git reports as changed instead of scanning everything.
|
||||
filesChecked = gitChanges.modified.length + gitChanges.added.length + gitChanges.deleted.length;
|
||||
|
||||
// Handle deleted files
|
||||
for (const filePath of gitChanges.deleted) {
|
||||
const tracked = this.queries.getFileByPath(filePath);
|
||||
if (tracked) {
|
||||
this.queries.deleteFile(filePath);
|
||||
filesRemoved++;
|
||||
}
|
||||
}
|
||||
|
||||
const contentHash = hashContent(content);
|
||||
const tracked = trackedFiles.find((f) => f.path === filePath);
|
||||
// Handle modified files — read + hash only these files
|
||||
for (const filePath of gitChanges.modified) {
|
||||
const fullPath = path.join(this.rootDir, filePath);
|
||||
let content: string;
|
||||
try {
|
||||
content = fs.readFileSync(fullPath, 'utf-8');
|
||||
} catch (error) {
|
||||
captureException(error, { operation: 'sync-read-file', filePath });
|
||||
logDebug('Skipping unreadable file during sync', { filePath, error: String(error) });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!tracked) {
|
||||
// New file
|
||||
const contentHash = hashContent(content);
|
||||
const tracked = this.queries.getFileByPath(filePath);
|
||||
|
||||
if (!tracked) {
|
||||
filesToIndex.push(filePath);
|
||||
changedFilePaths.push(filePath);
|
||||
filesAdded++;
|
||||
} else if (tracked.contentHash !== contentHash) {
|
||||
filesToIndex.push(filePath);
|
||||
changedFilePaths.push(filePath);
|
||||
filesModified++;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle added (untracked) files
|
||||
for (const filePath of gitChanges.added) {
|
||||
filesToIndex.push(filePath);
|
||||
changedFilePaths.push(filePath);
|
||||
filesAdded++;
|
||||
} else if (tracked.contentHash !== contentHash) {
|
||||
// Modified file
|
||||
filesToIndex.push(filePath);
|
||||
filesModified++;
|
||||
}
|
||||
} else {
|
||||
// === Fallback: full scan (non-git project or git failure) ===
|
||||
const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
|
||||
filesChecked = currentFiles.size;
|
||||
|
||||
// Build Map for O(1) lookups instead of .find() per file
|
||||
const trackedFiles = this.queries.getAllFiles();
|
||||
const trackedMap = new Map<string, FileRecord>();
|
||||
for (const f of trackedFiles) {
|
||||
trackedMap.set(f.path, f);
|
||||
}
|
||||
|
||||
// Find files to remove (in DB but not on disk)
|
||||
for (const tracked of trackedFiles) {
|
||||
if (!currentFiles.has(tracked.path)) {
|
||||
this.queries.deleteFile(tracked.path);
|
||||
filesRemoved++;
|
||||
}
|
||||
}
|
||||
|
||||
// Find files to add or update
|
||||
for (const filePath of currentFiles) {
|
||||
const fullPath = path.join(this.rootDir, filePath);
|
||||
let content: string;
|
||||
try {
|
||||
content = fs.readFileSync(fullPath, 'utf-8');
|
||||
} catch (error) {
|
||||
captureException(error, { operation: 'sync-read-file', filePath });
|
||||
logDebug('Skipping unreadable file during sync', { filePath, error: String(error) });
|
||||
continue;
|
||||
}
|
||||
|
||||
const contentHash = hashContent(content);
|
||||
const tracked = trackedMap.get(filePath);
|
||||
|
||||
if (!tracked) {
|
||||
filesToIndex.push(filePath);
|
||||
changedFilePaths.push(filePath);
|
||||
filesAdded++;
|
||||
} else if (tracked.contentHash !== contentHash) {
|
||||
filesToIndex.push(filePath);
|
||||
changedFilePaths.push(filePath);
|
||||
filesModified++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -692,16 +816,71 @@ export class ExtractionOrchestrator {
|
||||
filesRemoved,
|
||||
nodesUpdated,
|
||||
durationMs: Date.now() - startTime,
|
||||
changedFilePaths: changedFilePaths.length > 0 ? changedFilePaths : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get files that have changed since last index
|
||||
* Get files that have changed since last index.
|
||||
* Uses git status as a fast path when available, falling back to full scan.
|
||||
*/
|
||||
getChangedFiles(): { added: string[]; modified: string[]; removed: string[] } {
|
||||
const gitChanges = getGitChangedFiles(this.rootDir, this.config);
|
||||
|
||||
if (gitChanges) {
|
||||
// === Git fast path ===
|
||||
const added: string[] = [];
|
||||
const modified: string[] = [];
|
||||
const removed: string[] = [];
|
||||
|
||||
// Deleted files — only report if tracked in DB
|
||||
for (const filePath of gitChanges.deleted) {
|
||||
const tracked = this.queries.getFileByPath(filePath);
|
||||
if (tracked) {
|
||||
removed.push(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
// Modified files — read + hash only these, compare with DB
|
||||
for (const filePath of gitChanges.modified) {
|
||||
const fullPath = path.join(this.rootDir, filePath);
|
||||
let content: string;
|
||||
try {
|
||||
content = fs.readFileSync(fullPath, 'utf-8');
|
||||
} catch (error) {
|
||||
captureException(error, { operation: 'detect-changes-read-file', filePath });
|
||||
logDebug('Skipping unreadable file while detecting changes', { filePath, error: String(error) });
|
||||
continue;
|
||||
}
|
||||
|
||||
const contentHash = hashContent(content);
|
||||
const tracked = this.queries.getFileByPath(filePath);
|
||||
|
||||
if (!tracked) {
|
||||
added.push(filePath);
|
||||
} else if (tracked.contentHash !== contentHash) {
|
||||
modified.push(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
// Added (untracked) files
|
||||
for (const filePath of gitChanges.added) {
|
||||
added.push(filePath);
|
||||
}
|
||||
|
||||
return { added, modified, removed };
|
||||
}
|
||||
|
||||
// === Fallback: full scan (non-git project or git failure) ===
|
||||
const currentFiles = new Set(scanDirectory(this.rootDir, this.config));
|
||||
const trackedFiles = this.queries.getAllFiles();
|
||||
|
||||
// Build Map for O(1) lookups
|
||||
const trackedMap = new Map<string, FileRecord>();
|
||||
for (const f of trackedFiles) {
|
||||
trackedMap.set(f.path, f);
|
||||
}
|
||||
|
||||
const added: string[] = [];
|
||||
const modified: string[] = [];
|
||||
const removed: string[] = [];
|
||||
@@ -726,7 +905,7 @@ export class ExtractionOrchestrator {
|
||||
}
|
||||
|
||||
const contentHash = hashContent(content);
|
||||
const tracked = trackedFiles.find((f) => f.path === filePath);
|
||||
const tracked = trackedMap.get(filePath);
|
||||
|
||||
if (!tracked) {
|
||||
added.push(filePath);
|
||||
@@ -741,4 +920,4 @@ export class ExtractionOrchestrator {
|
||||
|
||||
// Re-export useful types and functions
|
||||
export { extractFromSource } from './tree-sitter';
|
||||
export { detectLanguage, isLanguageSupported, getSupportedLanguages } from './grammars';
|
||||
export { detectLanguage, isLanguageSupported, getSupportedLanguages, initGrammars } from './grammars';
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* Handles parsing source code and extracting structural information.
|
||||
*/
|
||||
|
||||
import { SyntaxNode, Tree } from 'tree-sitter';
|
||||
import { Node as SyntaxNode, Tree } from 'web-tree-sitter';
|
||||
import * as crypto from 'crypto';
|
||||
import * as path from 'path';
|
||||
import {
|
||||
@@ -934,7 +934,10 @@ export class TreeSitterExtractor {
|
||||
}
|
||||
|
||||
try {
|
||||
this.tree = parser.parse(this.source);
|
||||
this.tree = parser.parse(this.source) ?? null;
|
||||
if (!this.tree) {
|
||||
throw new Error('Parser returned null tree');
|
||||
}
|
||||
|
||||
// Create file node representing the source file
|
||||
const fileNode: Node = {
|
||||
@@ -1775,9 +1778,15 @@ export class TreeSitterExtractor {
|
||||
if (namespacePrefix && useGroup) {
|
||||
// Grouped import - create one import per item
|
||||
const prefix = getNodeText(namespacePrefix, this.source);
|
||||
const useClauses = useGroup.namedChildren.filter((c: SyntaxNode) => c.type === 'namespace_use_clause');
|
||||
const useClauses = useGroup.namedChildren.filter((c: SyntaxNode) =>
|
||||
c.type === 'namespace_use_group_clause' || c.type === 'namespace_use_clause'
|
||||
);
|
||||
for (const clause of useClauses) {
|
||||
const name = clause.namedChildren.find((c: SyntaxNode) => c.type === 'name');
|
||||
// WASM grammar wraps names in namespace_name; native uses name directly
|
||||
const nsName = clause.namedChildren.find((c: SyntaxNode) => c.type === 'namespace_name');
|
||||
const name = nsName
|
||||
? nsName.namedChildren.find((c: SyntaxNode) => c.type === 'name')
|
||||
: clause.namedChildren.find((c: SyntaxNode) => c.type === 'name');
|
||||
if (name) {
|
||||
const fullPath = `${prefix}\\${getNodeText(name, this.source)}`;
|
||||
this.createNode('import', fullPath, node, {
|
||||
|
||||
Executable
BIN
Binary file not shown.
Reference in New Issue
Block a user