diff --git a/__tests__/sync.test.ts b/__tests__/sync.test.ts index 909c079..8365f63 100644 --- a/__tests__/sync.test.ts +++ b/__tests__/sync.test.ts @@ -10,6 +10,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { execFileSync } from 'child_process'; import CodeGraph from '../src/index'; describe('Sync Module', () => { @@ -150,4 +151,112 @@ describe('Sync Module', () => { }); }); }); + + describe('Git-based sync', () => { + let testDir: string; + let cg: CodeGraph; + + function git(...args: string[]) { + execFileSync('git', args, { cwd: testDir, stdio: 'pipe' }); + } + + beforeEach(async () => { + testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-git-sync-')); + + // Initialize a git repo with an initial commit + git('init'); + git('config', 'user.email', 'test@test.com'); + git('config', 'user.name', 'Test'); + + const srcDir = path.join(testDir, 'src'); + fs.mkdirSync(srcDir); + fs.writeFileSync( + path.join(srcDir, 'index.ts'), + `export function hello() { return 'world'; }` + ); + + git('add', '-A'); + git('commit', '-m', 'initial'); + + // Initialize CodeGraph and index + cg = CodeGraph.initSync(testDir, { + config: { + include: ['**/*.ts'], + exclude: [], + }, + }); + await cg.indexAll(); + }); + + afterEach(() => { + if (cg) { + cg.destroy(); + } + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true, force: true }); + } + }); + + it('should detect modified files via git', async () => { + fs.writeFileSync( + path.join(testDir, 'src', 'index.ts'), + `export function hello() { return 'modified'; }` + ); + + const result = await cg.sync(); + + expect(result.filesModified).toBe(1); + expect(result.changedFilePaths).toContain('src/index.ts'); + }); + + it('should detect new untracked files via git', async () => { + fs.writeFileSync( + path.join(testDir, 'src', 'new.ts'), + `export function newFunc() { return 42; }` + ); + + const result = await cg.sync(); + + expect(result.filesAdded).toBe(1); + expect(result.changedFilePaths).toContain('src/new.ts'); + + // Verify the function was indexed + const nodes = cg.searchNodes('newFunc'); + expect(nodes.length).toBeGreaterThan(0); + }); + + it('should detect deleted files via git', async () => { + fs.unlinkSync(path.join(testDir, 'src', 'index.ts')); + + const result = await cg.sync(); + + expect(result.filesRemoved).toBe(1); + + // Verify function is gone + const nodes = cg.searchNodes('hello'); + expect(nodes.length).toBe(0); + }); + + it('should skip files not matching config', async () => { + // Create a .js file which doesn't match **/*.ts + fs.writeFileSync( + path.join(testDir, 'src', 'ignored.js'), + `function ignored() {}` + ); + + const result = await cg.sync(); + + expect(result.filesAdded).toBe(0); + expect(result.filesModified).toBe(0); + }); + + it('should report no changes on clean working tree', async () => { + const result = await cg.sync(); + + expect(result.filesAdded).toBe(0); + expect(result.filesModified).toBe(0); + expect(result.filesRemoved).toBe(0); + expect(result.changedFilePaths).toBeUndefined(); + }); + }); }); diff --git a/package-lock.json b/package-lock.json index 668eedf..cfbad97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@colbymchenry/codegraph", - "version": "0.5.1", + "version": "0.5.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@colbymchenry/codegraph", - "version": "0.5.1", + "version": "0.5.2", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 0da7670..0fe7757 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@colbymchenry/codegraph", - "version": "0.5.1", + "version": "0.5.2", "description": "Supercharge Claude Code with semantic code intelligence. 30% fewer tokens, 25% fewer tool calls, 100% local.", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/src/db/queries.ts b/src/db/queries.ts index aed2a96..83702de 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -896,6 +896,30 @@ export class QueryBuilder { })); } + /** + * Get unresolved references scoped to specific file paths. + * Uses the idx_unresolved_file_path index for efficient lookup. + */ + getUnresolvedReferencesByFiles(filePaths: string[]): UnresolvedReference[] { + if (filePaths.length === 0) return []; + + const placeholders = filePaths.map(() => '?').join(','); + const rows = this.db + .prepare(`SELECT * FROM unresolved_refs WHERE file_path IN (${placeholders})`) + .all(...filePaths) as UnresolvedRefRow[]; + + return rows.map((row) => ({ + fromNodeId: row.from_node_id, + referenceName: row.reference_name, + referenceKind: row.reference_kind as EdgeKind, + line: row.line, + column: row.col, + candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, + filePath: row.file_path, + language: row.language as Language, + })); + } + /** * Delete all unresolved references (after resolution) */ diff --git a/src/extraction/index.ts b/src/extraction/index.ts index b3e1b2c..4573bb6 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -63,6 +63,7 @@ export interface SyncResult { filesRemoved: number; nodesUpdated: number; durationMs: number; + changedFilePaths?: string[]; } /** @@ -105,27 +106,80 @@ export function shouldIncludeFile( } /** - * Get directories ignored by .gitignore using git ls-files. - * Returns a Set of normalized relative directory paths (forward slashes, no trailing slash). - * Gracefully returns empty Set on any failure. + * Get all files visible to git (tracked + untracked but not ignored). + * Respects .gitignore at all levels (root, subdirectories). + * Returns null on failure (non-git project) so callers can fall back. */ -function getGitIgnoredDirectories(rootDir: string): Set { +function getGitVisibleFiles(rootDir: string): Set | null { + try { + // -c = cached (tracked), -o = others (untracked), --exclude-standard = respect .gitignore + const output = execFileSync( + 'git', + ['ls-files', '-co', '--exclude-standard'], + { cwd: rootDir, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] } + ); + const files = new Set(); + for (const line of output.split('\n')) { + const trimmed = line.trim(); + if (trimmed) { + files.add(normalizePath(trimmed)); + } + } + return files; + } catch { + return null; + } +} + +/** + * Result of git-based change detection. + * Returns null when git is unavailable (non-git project or command failure), + * signaling the caller to fall back to full filesystem scan. + */ +interface GitChanges { + modified: string[]; // M, MM, AM — files to re-hash + re-index + added: string[]; // ?? — new untracked files to index + deleted: string[]; // D — files to remove from DB +} + +/** + * Use `git status` to detect changed files instead of scanning every file. + * Returns null on failure so callers fall back to full scan. + */ +function getGitChangedFiles(rootDir: string, config: CodeGraphConfig): GitChanges | null { try { const output = execFileSync( 'git', - ['ls-files', '-oi', '--exclude-standard', '--directory'], + ['status', '--porcelain', '--no-renames'], { cwd: rootDir, encoding: 'utf-8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'] } ); - const dirs = new Set(); + + const modified: string[] = []; + const added: string[] = []; + const deleted: string[] = []; + for (const line of output.split('\n')) { - const trimmed = line.trim(); - if (trimmed.endsWith('/')) { - dirs.add(normalizePath(trimmed.slice(0, -1))); + if (line.length < 4) continue; // Minimum: "XY file" + + const statusCode = line.substring(0, 2); + const filePath = normalizePath(line.substring(3)); + + // Skip files that don't match include/exclude config + if (!shouldIncludeFile(filePath, config)) continue; + + if (statusCode === '??') { + added.push(filePath); + } else if (statusCode.includes('D')) { + deleted.push(filePath); + } else { + // M, MM, AM, A (staged), etc. — treat as modified + modified.push(filePath); } } - return dirs; + + return { modified, added, deleted }; } catch { - return new Set(); + return null; } } @@ -135,21 +189,49 @@ function getGitIgnoredDirectories(rootDir: string): Set { const CODEGRAPH_IGNORE_MARKER = '.codegraphignore'; /** - * Recursively scan directory for source files + * Recursively scan directory for source files. + * + * In git repos, uses `git ls-files` to get the file list (inherently + * respects .gitignore at all levels), then filters by config include patterns. + * Falls back to filesystem walk for non-git projects. */ export function scanDirectory( rootDir: string, config: CodeGraphConfig, onProgress?: (current: number, file: string) => void +): string[] { + // Fast path: use git to get all visible files (respects .gitignore everywhere) + const gitFiles = getGitVisibleFiles(rootDir); + if (gitFiles) { + const files: string[] = []; + let count = 0; + for (const filePath of gitFiles) { + if (shouldIncludeFile(filePath, config)) { + files.push(filePath); + count++; + onProgress?.(count, filePath); + } + } + return files; + } + + // Fallback: walk filesystem for non-git projects + return scanDirectoryWalk(rootDir, config, onProgress); +} + +/** + * Filesystem walk fallback for non-git projects. + */ +function scanDirectoryWalk( + rootDir: string, + config: CodeGraphConfig, + onProgress?: (current: number, file: string) => void ): string[] { const files: string[] = []; let count = 0; - // Track visited real paths to detect symlink cycles const visitedDirs = new Set(); - const gitIgnoredDirs = getGitIgnoredDirectories(rootDir); function walk(dir: string): void { - // Resolve real path to detect symlink cycles let realDir: string; try { realDir = fs.realpathSync(dir); @@ -164,7 +246,7 @@ export function scanDirectory( } visitedDirs.add(realDir); - // Check for .codegraphignore marker file - skip entire directory tree if present + // Check for .codegraphignore marker file const ignoreMarker = path.join(dir, CODEGRAPH_IGNORE_MARKER); if (fs.existsSync(ignoreMarker)) { logDebug('Skipping directory due to .codegraphignore marker', { dir }); @@ -184,17 +266,11 @@ export function scanDirectory( const fullPath = path.join(dir, entry.name); const relativePath = normalizePath(path.relative(rootDir, fullPath)); - // Follow symlinked directories, but skip symlinked files to non-project targets if (entry.isSymbolicLink()) { try { const realTarget = fs.realpathSync(fullPath); const stat = fs.statSync(realTarget); if (stat.isDirectory()) { - // Check gitignore first (fast O(1) lookup) - if (gitIgnoredDirs.has(relativePath)) { - continue; - } - // Check exclusion, then recurse (cycle detection handles the rest) const dirPattern = relativePath + '/'; let excluded = false; for (const pattern of config.exclude) { @@ -210,9 +286,7 @@ export function scanDirectory( if (shouldIncludeFile(relativePath, config)) { files.push(relativePath); count++; - if (onProgress) { - onProgress(count, relativePath); - } + onProgress?.(count, relativePath); } } } catch { @@ -222,11 +296,6 @@ export function scanDirectory( } if (entry.isDirectory()) { - // Check gitignore first (fast O(1) lookup) - if (gitIgnoredDirs.has(relativePath)) { - continue; - } - // Check if directory should be excluded const dirPattern = relativePath + '/'; let excluded = false; for (const pattern of config.exclude) { @@ -242,9 +311,7 @@ export function scanDirectory( if (shouldIncludeFile(relativePath, config)) { files.push(relativePath); count++; - if (onProgress) { - onProgress(count, relativePath); - } + onProgress?.(count, relativePath); } } } @@ -611,7 +678,8 @@ export class ExtractionOrchestrator { } /** - * Sync with current file state + * Sync with current file state. + * Uses git status as a fast path when available, falling back to full scan. */ async sync(onProgress?: (progress: IndexProgress) => void): Promise { const startTime = Date.now(); @@ -620,53 +688,107 @@ export class ExtractionOrchestrator { let filesModified = 0; let filesRemoved = 0; let nodesUpdated = 0; + const changedFilePaths: string[] = []; - // Get current files on disk onProgress?.({ phase: 'scanning', current: 0, total: 0, }); - const currentFiles = new Set(scanDirectory(this.rootDir, this.config)); - filesChecked = currentFiles.size; - - // Get tracked files from database - const trackedFiles = this.queries.getAllFiles(); - - // Find files to remove (in DB but not on disk) - for (const tracked of trackedFiles) { - if (!currentFiles.has(tracked.path)) { - this.queries.deleteFile(tracked.path); - filesRemoved++; - } - } - - // Find files to add or update const filesToIndex: string[] = []; + const gitChanges = getGitChangedFiles(this.rootDir, this.config); - for (const filePath of currentFiles) { - const fullPath = path.join(this.rootDir, filePath); - let content: string; - try { - content = fs.readFileSync(fullPath, 'utf-8'); - } catch (error) { - captureException(error, { operation: 'sync-read-file', filePath }); - logDebug('Skipping unreadable file during sync', { filePath, error: String(error) }); - continue; + if (gitChanges) { + // === Git fast path === + // Only inspect the files git reports as changed instead of scanning everything. + filesChecked = gitChanges.modified.length + gitChanges.added.length + gitChanges.deleted.length; + + // Handle deleted files + for (const filePath of gitChanges.deleted) { + const tracked = this.queries.getFileByPath(filePath); + if (tracked) { + this.queries.deleteFile(filePath); + filesRemoved++; + } } - const contentHash = hashContent(content); - const tracked = trackedFiles.find((f) => f.path === filePath); + // Handle modified files — read + hash only these files + for (const filePath of gitChanges.modified) { + const fullPath = path.join(this.rootDir, filePath); + let content: string; + try { + content = fs.readFileSync(fullPath, 'utf-8'); + } catch (error) { + captureException(error, { operation: 'sync-read-file', filePath }); + logDebug('Skipping unreadable file during sync', { filePath, error: String(error) }); + continue; + } - if (!tracked) { - // New file + const contentHash = hashContent(content); + const tracked = this.queries.getFileByPath(filePath); + + if (!tracked) { + filesToIndex.push(filePath); + changedFilePaths.push(filePath); + filesAdded++; + } else if (tracked.contentHash !== contentHash) { + filesToIndex.push(filePath); + changedFilePaths.push(filePath); + filesModified++; + } + } + + // Handle added (untracked) files + for (const filePath of gitChanges.added) { filesToIndex.push(filePath); + changedFilePaths.push(filePath); filesAdded++; - } else if (tracked.contentHash !== contentHash) { - // Modified file - filesToIndex.push(filePath); - filesModified++; + } + } else { + // === Fallback: full scan (non-git project or git failure) === + const currentFiles = new Set(scanDirectory(this.rootDir, this.config)); + filesChecked = currentFiles.size; + + // Build Map for O(1) lookups instead of .find() per file + const trackedFiles = this.queries.getAllFiles(); + const trackedMap = new Map(); + for (const f of trackedFiles) { + trackedMap.set(f.path, f); + } + + // Find files to remove (in DB but not on disk) + for (const tracked of trackedFiles) { + if (!currentFiles.has(tracked.path)) { + this.queries.deleteFile(tracked.path); + filesRemoved++; + } + } + + // Find files to add or update + for (const filePath of currentFiles) { + const fullPath = path.join(this.rootDir, filePath); + let content: string; + try { + content = fs.readFileSync(fullPath, 'utf-8'); + } catch (error) { + captureException(error, { operation: 'sync-read-file', filePath }); + logDebug('Skipping unreadable file during sync', { filePath, error: String(error) }); + continue; + } + + const contentHash = hashContent(content); + const tracked = trackedMap.get(filePath); + + if (!tracked) { + filesToIndex.push(filePath); + changedFilePaths.push(filePath); + filesAdded++; + } else if (tracked.contentHash !== contentHash) { + filesToIndex.push(filePath); + changedFilePaths.push(filePath); + filesModified++; + } } } @@ -692,16 +814,71 @@ export class ExtractionOrchestrator { filesRemoved, nodesUpdated, durationMs: Date.now() - startTime, + changedFilePaths: changedFilePaths.length > 0 ? changedFilePaths : undefined, }; } /** - * Get files that have changed since last index + * Get files that have changed since last index. + * Uses git status as a fast path when available, falling back to full scan. */ getChangedFiles(): { added: string[]; modified: string[]; removed: string[] } { + const gitChanges = getGitChangedFiles(this.rootDir, this.config); + + if (gitChanges) { + // === Git fast path === + const added: string[] = []; + const modified: string[] = []; + const removed: string[] = []; + + // Deleted files — only report if tracked in DB + for (const filePath of gitChanges.deleted) { + const tracked = this.queries.getFileByPath(filePath); + if (tracked) { + removed.push(filePath); + } + } + + // Modified files — read + hash only these, compare with DB + for (const filePath of gitChanges.modified) { + const fullPath = path.join(this.rootDir, filePath); + let content: string; + try { + content = fs.readFileSync(fullPath, 'utf-8'); + } catch (error) { + captureException(error, { operation: 'detect-changes-read-file', filePath }); + logDebug('Skipping unreadable file while detecting changes', { filePath, error: String(error) }); + continue; + } + + const contentHash = hashContent(content); + const tracked = this.queries.getFileByPath(filePath); + + if (!tracked) { + added.push(filePath); + } else if (tracked.contentHash !== contentHash) { + modified.push(filePath); + } + } + + // Added (untracked) files + for (const filePath of gitChanges.added) { + added.push(filePath); + } + + return { added, modified, removed }; + } + + // === Fallback: full scan (non-git project or git failure) === const currentFiles = new Set(scanDirectory(this.rootDir, this.config)); const trackedFiles = this.queries.getAllFiles(); + // Build Map for O(1) lookups + const trackedMap = new Map(); + for (const f of trackedFiles) { + trackedMap.set(f.path, f); + } + const added: string[] = []; const modified: string[] = []; const removed: string[] = []; @@ -726,7 +903,7 @@ export class ExtractionOrchestrator { } const contentHash = hashContent(content); - const tracked = trackedFiles.find((f) => f.path === filePath); + const tracked = trackedMap.get(filePath); if (!tracked) { added.push(filePath); diff --git a/src/index.ts b/src/index.ts index 5bd43a2..4758e91 100644 --- a/src/index.ts +++ b/src/index.ts @@ -449,15 +449,18 @@ export class CodeGraph { // Resolve references if files were updated if (result.filesAdded > 0 || result.filesModified > 0) { - const unresolvedCount = this.queries.getUnresolvedReferences().length; + // Scope resolution to changed files when available (git fast path) + const unresolvedRefs = result.changedFilePaths + ? this.queries.getUnresolvedReferencesByFiles(result.changedFilePaths) + : this.queries.getUnresolvedReferences(); options.onProgress?.({ phase: 'resolving', current: 0, - total: unresolvedCount, + total: unresolvedRefs.length, }); - this.resolveReferences((current, total) => { + this.resolver.resolveAndPersist(unresolvedRefs, (current, total) => { options.onProgress?.({ phase: 'resolving', current,