From 62a6cf38feafb90aef94a1216aa5e73f25a8a05e Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 10 Feb 2026 04:08:10 -0600 Subject: [PATCH 1/5] Fix .gitignore support and Windows path separator bug in scanner Normalize paths to forward slashes in matchesGlob() and scanDirectory() so glob exclude patterns work on Windows. Add getGitIgnoredDirectories() using git ls-files to skip .gitignore'd directories during indexing. Co-Authored-By: Claude Opus 4.6 --- __tests__/extraction.test.ts | 107 ++++++++++++++++++++++++++++++++++- src/config.ts | 4 ++ src/extraction/index.ts | 42 +++++++++++++- src/utils.ts | 8 +++ 4 files changed, 158 insertions(+), 3 deletions(-) diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index 3e72ea6..b2ccca3 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -9,8 +9,10 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import { CodeGraph } from '../src'; -import { extractFromSource } from '../src/extraction'; +import { extractFromSource, scanDirectory, shouldIncludeFile } from '../src/extraction'; import { detectLanguage, isLanguageSupported, getSupportedLanguages } from '../src/extraction/grammars'; +import { normalizePath } from '../src/utils'; +import { DEFAULT_CONFIG } from '../src/types'; // Create a temporary directory for each test function createTempDir(): string { @@ -1880,3 +1882,106 @@ export function multiply(a: number, b: number): number { cg.close(); }); }); + +describe('Path Normalization', () => { + it('should convert backslashes to forward slashes', () => { + expect(normalizePath('gui\\node_modules\\foo')).toBe('gui/node_modules/foo'); + expect(normalizePath('src\\components\\Button.tsx')).toBe('src/components/Button.tsx'); + }); + + it('should leave forward-slash paths unchanged', () => { + expect(normalizePath('src/components/Button.tsx')).toBe('src/components/Button.tsx'); + }); + + it('should handle empty string', () => { + expect(normalizePath('')).toBe(''); + }); +}); + +describe('Directory Exclusion', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = createTempDir(); + }); + + afterEach(() => { + cleanupTempDir(tempDir); + }); + + it('should exclude node_modules directories', () => { + // Create structure: src/index.ts + node_modules/pkg/index.js + const srcDir = path.join(tempDir, 'src'); + const nmDir = path.join(tempDir, 'node_modules', 'pkg'); + fs.mkdirSync(srcDir, { recursive: true }); + fs.mkdirSync(nmDir, { recursive: true }); + fs.writeFileSync(path.join(srcDir, 'index.ts'), 'export const x = 1;'); + fs.writeFileSync(path.join(nmDir, 'index.js'), 'module.exports = {};'); + + const config = { ...DEFAULT_CONFIG, rootDir: tempDir }; + const files = scanDirectory(tempDir, config); + + expect(files).toContain('src/index.ts'); + expect(files.every((f) => !f.includes('node_modules'))).toBe(true); + }); + + it('should exclude nested node_modules directories', () => { + // Create structure: packages/app/node_modules/pkg/index.js + const srcDir = path.join(tempDir, 'packages', 'app', 'src'); + const nmDir = path.join(tempDir, 'packages', 'app', 'node_modules', 'pkg'); + fs.mkdirSync(srcDir, { recursive: true }); + fs.mkdirSync(nmDir, { recursive: true }); + fs.writeFileSync(path.join(srcDir, 'index.ts'), 'export const x = 1;'); + fs.writeFileSync(path.join(nmDir, 'index.js'), 'module.exports = {};'); + + const config = { ...DEFAULT_CONFIG, rootDir: tempDir }; + const files = scanDirectory(tempDir, config); + + expect(files).toContain('packages/app/src/index.ts'); + expect(files.every((f) => !f.includes('node_modules'))).toBe(true); + }); + + it('should exclude .git directories', () => { + const srcDir = path.join(tempDir, 'src'); + const gitDir = path.join(tempDir, '.git', 'objects'); + fs.mkdirSync(srcDir, { recursive: true }); + fs.mkdirSync(gitDir, { recursive: true }); + fs.writeFileSync(path.join(srcDir, 'index.ts'), 'export const x = 1;'); + fs.writeFileSync(path.join(gitDir, 'pack.ts'), 'export const y = 2;'); + + const config = { ...DEFAULT_CONFIG, rootDir: tempDir }; + const files = scanDirectory(tempDir, config); + + expect(files).toContain('src/index.ts'); + expect(files.every((f) => !f.includes('.git'))).toBe(true); + }); + + it('should return forward-slash paths on all platforms', () => { + const srcDir = path.join(tempDir, 'src', 'components'); + fs.mkdirSync(srcDir, { recursive: true }); + fs.writeFileSync(path.join(srcDir, 'Button.tsx'), 'export function Button() {}'); + + const config = { ...DEFAULT_CONFIG, rootDir: tempDir }; + const files = scanDirectory(tempDir, config); + + expect(files.length).toBe(1); + expect(files[0]).toBe('src/components/Button.tsx'); + expect(files[0]).not.toContain('\\'); + }); + + it('should respect .codegraphignore marker', () => { + const srcDir = path.join(tempDir, 'src'); + const vendorDir = path.join(tempDir, 'vendor'); + fs.mkdirSync(srcDir, { recursive: true }); + fs.mkdirSync(vendorDir, { recursive: true }); + fs.writeFileSync(path.join(srcDir, 'index.ts'), 'export const x = 1;'); + fs.writeFileSync(path.join(vendorDir, 'lib.ts'), 'export const y = 2;'); + fs.writeFileSync(path.join(vendorDir, '.codegraphignore'), ''); + + const config = { ...DEFAULT_CONFIG, rootDir: tempDir }; + const files = scanDirectory(tempDir, config); + + expect(files).toContain('src/index.ts'); + expect(files.every((f) => !f.includes('vendor'))).toBe(true); + }); +}); diff --git a/src/config.ts b/src/config.ts index 4808e6c..6409f31 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,6 +7,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { CodeGraphConfig, DEFAULT_CONFIG, Language, NodeKind } from './types'; +import { normalizePath } from './utils'; /** * Configuration filename @@ -240,6 +241,9 @@ export function addCustomPattern( * Check if a file path matches the include/exclude patterns */ export function shouldIncludeFile(filePath: string, config: CodeGraphConfig): boolean { + // Normalize to forward slashes so Windows backslash paths match glob patterns + filePath = normalizePath(filePath); + // Simple glob matching (for now, just check if any pattern matches) // A full implementation would use a proper glob library diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 3fa6fc9..fb6cc21 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -8,6 +8,7 @@ import * as fs from 'fs'; import * as fsp from 'fs/promises'; import * as path from 'path'; import * as crypto from 'crypto'; +import { execFileSync } from 'child_process'; import { Language, FileRecord, @@ -20,7 +21,7 @@ import { extractFromSource } from './tree-sitter'; import { detectLanguage, isLanguageSupported } from './grammars'; import { logDebug } from '../errors'; import { captureException } from '../sentry'; -import { validatePathWithinRoot } from '../utils'; +import { validatePathWithinRoot, normalizePath } from '../utils'; /** * Progress callback for indexing operations @@ -68,6 +69,9 @@ export function hashContent(content: string): string { * Check if a path matches any glob pattern (simplified) */ function matchesGlob(filePath: string, pattern: string): boolean { + // Normalize to forward slashes so Windows backslash paths match glob patterns + filePath = normalizePath(filePath); + // Convert glob to regex using placeholders to avoid conflicts let regexStr = pattern; @@ -114,6 +118,31 @@ export function shouldIncludeFile( return false; } +/** + * Get directories ignored by .gitignore using git ls-files. + * Returns a Set of normalized relative directory paths (forward slashes, no trailing slash). + * Gracefully returns empty Set on any failure. + */ +function getGitIgnoredDirectories(rootDir: string): Set { + try { + const output = execFileSync( + 'git', + ['ls-files', '-oi', '--exclude-standard', '--directory'], + { cwd: rootDir, encoding: 'utf-8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'] } + ); + const dirs = new Set(); + for (const line of output.split('\n')) { + const trimmed = line.trim(); + if (trimmed.endsWith('/')) { + dirs.add(normalizePath(trimmed.slice(0, -1))); + } + } + return dirs; + } catch { + return new Set(); + } +} + /** * Marker file name that indicates a directory (and all children) should be skipped */ @@ -130,6 +159,7 @@ export function scanDirectory( const files: string[] = []; let count = 0; const visitedRealPaths = new Set(); // Symlink cycle detection + const gitIgnoredDirs = getGitIgnoredDirectories(rootDir); function walk(dir: string): void { // Symlink cycle detection: resolve real path and skip if already visited @@ -163,7 +193,7 @@ export function scanDirectory( for (const entry of entries) { const fullPath = path.join(dir, entry.name); - const relativePath = path.relative(rootDir, fullPath); + const relativePath = normalizePath(path.relative(rootDir, fullPath)); // Follow symlinked directories, but skip symlinked files to non-project targets if (entry.isSymbolicLink()) { @@ -171,6 +201,10 @@ export function scanDirectory( const realTarget = fs.realpathSync(fullPath); const stat = fs.statSync(realTarget); if (stat.isDirectory()) { + // Check gitignore first (fast O(1) lookup) + if (gitIgnoredDirs.has(relativePath)) { + continue; + } // Check exclusion, then recurse (cycle detection handles the rest) const dirPattern = relativePath + '/'; let excluded = false; @@ -199,6 +233,10 @@ export function scanDirectory( } if (entry.isDirectory()) { + // Check gitignore first (fast O(1) lookup) + if (gitIgnoredDirs.has(relativePath)) { + continue; + } // Check if directory should be excluded const dirPattern = relativePath + '/'; let excluded = false; diff --git a/src/utils.ts b/src/utils.ts index a4f4e55..c406a80 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -74,6 +74,14 @@ export function clamp(value: number, min: number, max: number): number { return Math.max(min, Math.min(max, value)); } +/** + * Normalize a file path to use forward slashes. + * Fixes Windows backslash paths so glob matching works consistently. + */ +export function normalizePath(filePath: string): string { + return filePath.replace(/\\/g, '/'); +} + /** * Cross-process file lock using lock files. * Prevents concurrent database writes from CLI, MCP server, and git hooks. From c80378d73ca757c3b22277a92ea94c2ea00c8502 Mon Sep 17 00:00:00 2001 From: Martin Oehlert <453360+MO2k4@users.noreply.github.com> Date: Tue, 10 Feb 2026 11:40:41 +0100 Subject: [PATCH 2/5] feat: search query utilities and multi-signal scoring - Add src/search/query-utils.ts with extractSearchTerms, scorePathRelevance, kindBonus, detectApiIntent, inferRouteDirectories - Add multi-signal scoring to searchNodes (kind bonus + path relevance) - Improve FTS query sanitization (strip :^ chars, filter boolean operators) - Add comprehensive search tests --- __tests__/search.test.ts | 214 ++++++++++++++++++++++++++++++++++++++ src/db/queries.ts | 14 ++- src/search/query-utils.ts | 129 +++++++++++++++++++++++ 3 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 __tests__/search.test.ts create mode 100644 src/search/query-utils.ts diff --git a/__tests__/search.test.ts b/__tests__/search.test.ts new file mode 100644 index 0000000..25b7c26 --- /dev/null +++ b/__tests__/search.test.ts @@ -0,0 +1,214 @@ +/** + * Search Query Utilities Tests + * + * Tests multi-signal scoring, kind bonuses, path relevance, and API intent detection. + */ + +import { describe, it, expect } from 'vitest'; +import { + extractSearchTerms, + scorePathRelevance, + kindBonus, + detectApiIntent, + inferRouteDirectories, + STOP_WORDS, +} from '../src/search/query-utils'; + +describe('Search Query Utilities', () => { + describe('extractSearchTerms', () => { + it('should extract meaningful terms from a query', () => { + const terms = extractSearchTerms('find the login handler'); + expect(terms).toContain('login'); + expect(terms).toContain('handler'); + // 'find' and 'the' are stop words + expect(terms).not.toContain('find'); + expect(terms).not.toContain('the'); + }); + + it('should filter stop words', () => { + const terms = extractSearchTerms('how does the authentication work'); + expect(terms).not.toContain('how'); + expect(terms).not.toContain('does'); + expect(terms).not.toContain('the'); + expect(terms).toContain('authentication'); + expect(terms).toContain('work'); + }); + + it('should handle camelCase by lowercasing', () => { + const terms = extractSearchTerms('UserService'); + expect(terms).toContain('userservice'); + }); + + it('should strip punctuation', () => { + const terms = extractSearchTerms('payment.process()'); + expect(terms).toContain('payment'); + expect(terms).toContain('process'); + }); + + it('should return empty for all stop words', () => { + const terms = extractSearchTerms('how do I get the'); + expect(terms).toHaveLength(0); + }); + + it('should filter single-character terms', () => { + const terms = extractSearchTerms('a b c auth'); + expect(terms).toEqual(['auth']); + }); + }); + + describe('scorePathRelevance', () => { + it('should score filename matches highest', () => { + const score = scorePathRelevance('src/auth/login.ts', 'login'); + expect(score).toBeGreaterThanOrEqual(10); + }); + + it('should score directory matches', () => { + const score = scorePathRelevance('src/auth/index.ts', 'auth'); + expect(score).toBeGreaterThanOrEqual(5); + }); + + it('should return 0 for unrelated paths', () => { + const score = scorePathRelevance('src/utils/format.ts', 'payment'); + expect(score).toBe(0); + }); + + it('should accumulate scores for multiple matching terms', () => { + const score = scorePathRelevance('src/auth/login.ts', 'auth login'); + // Both 'auth' (dir match) and 'login' (filename match) + expect(score).toBeGreaterThanOrEqual(15); + }); + + it('should return 0 for empty query terms', () => { + const score = scorePathRelevance('src/auth/login.ts', 'the a an'); + expect(score).toBe(0); + }); + }); + + describe('kindBonus', () => { + it('should give functions and methods highest bonus', () => { + expect(kindBonus('function')).toBe(10); + expect(kindBonus('method')).toBe(10); + }); + + it('should rank functions > classes > variables > imports', () => { + expect(kindBonus('function')).toBeGreaterThan(kindBonus('class')); + expect(kindBonus('class')).toBeGreaterThan(kindBonus('variable')); + expect(kindBonus('variable')).toBeGreaterThan(kindBonus('import')); + }); + + it('should give routes high priority', () => { + expect(kindBonus('route')).toBeGreaterThanOrEqual(9); + }); + + it('should give components high priority', () => { + expect(kindBonus('component')).toBeGreaterThanOrEqual(8); + }); + + it('should return 0 for parameter and file kinds', () => { + expect(kindBonus('parameter')).toBe(0); + expect(kindBonus('file')).toBe(0); + }); + + it('should return 0 for unknown kinds', () => { + expect(kindBonus('unknown_kind' as any)).toBe(0); + }); + }); + + describe('detectApiIntent', () => { + it('should detect API-related queries', () => { + expect(detectApiIntent('find the API endpoint for users')).toBe(true); + expect(detectApiIntent('where is the login route')).toBe(true); + expect(detectApiIntent('show me the request handler')).toBe(true); + }); + + it('should detect HTTP method patterns', () => { + expect(detectApiIntent('GET /api/users')).toBe(true); + expect(detectApiIntent('post /users/create')).toBe(true); + }); + + it('should detect REST and GraphQL', () => { + expect(detectApiIntent('REST API for payments')).toBe(true); + expect(detectApiIntent('GraphQL resolver for orders')).toBe(true); + }); + + it('should not detect non-API queries', () => { + expect(detectApiIntent('fix the login bug')).toBe(false); + expect(detectApiIntent('add dark mode support')).toBe(false); + }); + + it('should detect controller and middleware mentions', () => { + expect(detectApiIntent('find the auth controller')).toBe(true); + expect(detectApiIntent('CORS middleware configuration')).toBe(true); + }); + }); + + describe('inferRouteDirectories', () => { + it('should detect route directories', () => { + const files = [ + 'src/routes/auth.ts', + 'src/routes/users.ts', + 'src/utils/format.ts', + ]; + const dirs = inferRouteDirectories(files); + expect(dirs).toBeDefined(); + if (dirs) { + expect(dirs.some(d => d.includes('route'))).toBe(true); + } + }); + + it('should detect controller directories', () => { + const files = [ + 'src/controllers/AuthController.ts', + 'src/models/User.ts', + ]; + const dirs = inferRouteDirectories(files); + expect(dirs).toBeDefined(); + if (dirs) { + expect(dirs.some(d => d.includes('controller'))).toBe(true); + } + }); + + it('should detect api directories', () => { + const files = [ + 'src/api/v1/users.ts', + 'src/api/v1/orders.ts', + ]; + const dirs = inferRouteDirectories(files); + expect(dirs).toBeDefined(); + if (dirs) { + expect(dirs.some(d => d.includes('api'))).toBe(true); + } + }); + + it('should return undefined when no route dirs found', () => { + const files = [ + 'src/utils/format.ts', + 'src/models/User.ts', + 'src/index.ts', + ]; + const dirs = inferRouteDirectories(files); + expect(dirs).toBeUndefined(); + }); + }); + + describe('STOP_WORDS', () => { + it('should contain common English stop words', () => { + expect(STOP_WORDS.has('the')).toBe(true); + expect(STOP_WORDS.has('and')).toBe(true); + expect(STOP_WORDS.has('or')).toBe(true); + }); + + it('should contain action verbs used in queries', () => { + expect(STOP_WORDS.has('find')).toBe(true); + expect(STOP_WORDS.has('show')).toBe(true); + expect(STOP_WORDS.has('get')).toBe(true); + expect(STOP_WORDS.has('list')).toBe(true); + }); + + it('should not contain technical terms', () => { + expect(STOP_WORDS.has('function')).toBe(false); + expect(STOP_WORDS.has('class')).toBe(false); + expect(STOP_WORDS.has('auth')).toBe(false); + }); + }); +}); diff --git a/src/db/queries.ts b/src/db/queries.ts index ffc0b7b..5d5bc43 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -18,6 +18,7 @@ import { SearchResult, } from '../types'; import { safeJsonParse } from '../utils'; +import { kindBonus, scorePathRelevance } from '../search/query-utils'; /** * Database row types (snake_case from SQLite) @@ -451,6 +452,15 @@ export class QueryBuilder { results = this.searchNodesLike(query, { kinds, languages, limit, offset }); } + // Apply multi-signal scoring + if (results.length > 0 && query) { + results = results.map(r => ({ + ...r, + score: r.score + kindBonus(r.node.kind) + scorePathRelevance(r.node.filePath, query), + })); + results.sort((a, b) => b.score - a.score); + } + return results; } @@ -463,9 +473,11 @@ export class QueryBuilder { // Add prefix wildcard for better matching (e.g., "auth" matches "AuthService", "authenticate") // Escape special FTS5 characters and add prefix wildcard const ftsQuery = query - .replace(/['"*()]/g, '') // Remove special chars + .replace(/['"*():^]/g, '') // Remove FTS5 special chars .split(/\s+/) .filter(term => term.length > 0) + // Strip FTS5 boolean operators to prevent query manipulation + .filter(term => !/^(AND|OR|NOT|NEAR)$/i.test(term)) .map(term => `"${term}"*`) // Prefix match each term .join(' OR '); diff --git a/src/search/query-utils.ts b/src/search/query-utils.ts new file mode 100644 index 0000000..e08d66e --- /dev/null +++ b/src/search/query-utils.ts @@ -0,0 +1,129 @@ +/** + * Search Query Utilities + * + * Shared module for search term extraction, scoring, and intent detection. + */ + +import * as path from 'path'; +import { Node } from '../types'; + +/** + * Common stop words to filter from search queries + */ +export const STOP_WORDS = new Set([ + 'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', + 'of', 'with', 'by', 'from', 'is', 'it', 'that', 'this', 'are', 'was', + 'be', 'has', 'had', 'have', 'do', 'does', 'did', 'will', 'would', 'could', + 'should', 'may', 'might', 'can', 'shall', 'not', 'no', 'all', 'each', + 'every', 'how', 'what', 'where', 'when', 'who', 'which', 'why', + 'i', 'me', 'my', 'we', 'our', 'you', 'your', 'he', 'she', 'they', + 'find', 'show', 'get', 'list', 'give', 'tell', +]); + +/** + * Extract meaningful search terms from a natural language query + */ +export function extractSearchTerms(query: string): string[] { + return query + .toLowerCase() + .replace(/[^\w\s-]/g, ' ') + .split(/\s+/) + .filter(term => term.length > 1 && !STOP_WORDS.has(term)); +} + +/** + * Score path relevance to a query + * Higher score = more relevant path + */ +export function scorePathRelevance(filePath: string, query: string): number { + const terms = extractSearchTerms(query); + if (terms.length === 0) return 0; + + const pathLower = filePath.toLowerCase(); + const fileName = path.basename(filePath).toLowerCase(); + const dirName = path.dirname(filePath).toLowerCase(); + let score = 0; + + for (const term of terms) { + // Exact filename match (strongest) + if (fileName.includes(term)) score += 10; + // Directory match + if (dirName.includes(term)) score += 5; + // General path match + else if (pathLower.includes(term)) score += 3; + } + + return score; +} + +/** + * Kind-based bonus for search ranking + * Functions and classes are typically more relevant than variables/imports + */ +export function kindBonus(kind: Node['kind']): number { + const bonuses: Record = { + function: 10, + method: 10, + class: 8, + interface: 7, + type_alias: 6, + struct: 6, + trait: 6, + enum: 5, + component: 8, + route: 9, + module: 4, + property: 3, + field: 3, + variable: 2, + constant: 3, + import: 1, + export: 1, + parameter: 0, + namespace: 4, + file: 0, + protocol: 6, + enum_member: 3, + }; + return bonuses[kind] ?? 0; +} + +/** + * Detect if a query has API/endpoint intent + */ +export function detectApiIntent(query: string): boolean { + const apiPatterns = [ + /\bapi\b/i, /\bendpoint/i, /\broute/i, /\bhandler/i, + /\bcontroller/i, /\bmiddleware/i, /\brest\b/i, /\bgraphql/i, + /\bget\s+\//, /\bpost\s+\//, /\bput\s+\//, /\bdelete\s+\//, + /\brequest/i, /\bresponse/i, /\bhttp/i, + ]; + return apiPatterns.some(p => p.test(query)); +} + +/** + * Infer route/controller directories from project structure + * Returns undefined if no route directories are detected + */ +export function inferRouteDirectories(files: string[]): string[] | undefined { + const routeDirs = new Set(); + const routePatterns = [ + /routes?\//i, /controllers?\//i, /handlers?\//i, + /api\//i, /endpoints?\//i, + ]; + + for (const file of files) { + for (const pattern of routePatterns) { + if (pattern.test(file)) { + const match = file.match(pattern); + if (match) { + const idx = file.indexOf(match[0]); + const dir = file.substring(0, idx + match[0].length - 1); + routeDirs.add(dir); + } + } + } + } + + return routeDirs.size > 0 ? Array.from(routeDirs) : undefined; +} From 0f2eda8da3e2f9e0ab8461ad11bd62fe8ea4fc50 Mon Sep 17 00:00:00 2001 From: Martin Oehlert <453360+MO2k4@users.noreply.github.com> Date: Tue, 10 Feb 2026 11:46:49 +0100 Subject: [PATCH 3/5] feat: file nodes, arrow function extraction, parallel I/O - Create file-kind nodes for each parsed source file - Add isInsideClassLikeNode() for method vs function detection - Extract arrow functions and function expressions from variable declarators - Batch file I/O with FILE_IO_BATCH_SIZE=10 using Promise.all - Add symlink cycle detection with visitedDirs Set in scanDirectory - Add lazy grammar loading with exported getGrammar() function - Add indexFileWithContent() for pre-read content processing - Add tests for file nodes and arrow function extraction --- __tests__/extraction.test.ts | 145 +++++++++++++++++++++++----- src/extraction/grammars.ts | 8 ++ src/extraction/index.ts | 172 ++++++++++++++++++++++++++++------ src/extraction/tree-sitter.ts | 115 ++++++++++++++++++++++- 4 files changed, 384 insertions(+), 56 deletions(-) diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index 3e72ea6..2c2f547 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -127,14 +127,19 @@ export function processPayment(amount: number): Promise { `; const result = extractFromSource('payment.ts', code); - expect(result.nodes).toHaveLength(1); - expect(result.nodes[0]).toMatchObject({ + // File node + function node + const fileNode = result.nodes.find((n) => n.kind === 'file'); + expect(fileNode).toBeDefined(); + expect(fileNode?.name).toBe('payment.ts'); + + const funcNode = result.nodes.find((n) => n.kind === 'function'); + expect(funcNode).toMatchObject({ kind: 'function', name: 'processPayment', language: 'typescript', isExported: true, }); - expect(result.nodes[0]?.signature).toContain('amount: number'); + expect(funcNode?.signature).toContain('amount: number'); }); it('should extract class declarations', () => { @@ -175,8 +180,11 @@ export interface User { `; const result = extractFromSource('types.ts', code); - expect(result.nodes).toHaveLength(1); - expect(result.nodes[0]).toMatchObject({ + const fileNode = result.nodes.find((n) => n.kind === 'file'); + expect(fileNode).toBeDefined(); + + const ifaceNode = result.nodes.find((n) => n.kind === 'interface'); + expect(ifaceNode).toMatchObject({ kind: 'interface', name: 'User', isExported: true, @@ -207,8 +215,9 @@ export const useAuth = (): AuthContextValue => { `; const result = extractFromSource('hooks.ts', code); - expect(result.nodes).toHaveLength(1); - expect(result.nodes[0]).toMatchObject({ + const funcNode = result.nodes.find((n) => n.kind === 'function' && n.name === 'useAuth'); + expect(funcNode).toBeDefined(); + expect(funcNode).toMatchObject({ kind: 'function', name: 'useAuth', isExported: true, @@ -223,8 +232,9 @@ export const processData = function(input: string): string { `; const result = extractFromSource('utils.ts', code); - expect(result.nodes).toHaveLength(1); - expect(result.nodes[0]).toMatchObject({ + const funcNode = result.nodes.find((n) => n.kind === 'function' && n.name === 'processData'); + expect(funcNode).toBeDefined(); + expect(funcNode).toMatchObject({ kind: 'function', name: 'processData', isExported: true, @@ -286,8 +296,9 @@ export const fetchData = async () => { `; const result = extractFromSource('api.js', code); - expect(result.nodes).toHaveLength(1); - expect(result.nodes[0]).toMatchObject({ + const funcNode = result.nodes.find((n) => n.kind === 'function' && n.name === 'fetchData'); + expect(funcNode).toBeDefined(); + expect(funcNode).toMatchObject({ kind: 'function', name: 'fetchData', isExported: true, @@ -306,8 +317,8 @@ export type AuthContextValue = { `; const result = extractFromSource('types.ts', code); - expect(result.nodes).toHaveLength(1); - expect(result.nodes[0]).toMatchObject({ + const typeNode = result.nodes.find((n) => n.kind === 'type_alias'); + expect(typeNode).toMatchObject({ kind: 'type_alias', name: 'AuthContextValue', isExported: true, @@ -323,8 +334,8 @@ type InternalState = { `; const result = extractFromSource('internal.ts', code); - expect(result.nodes).toHaveLength(1); - expect(result.nodes[0]).toMatchObject({ + const typeNode = result.nodes.find((n) => n.kind === 'type_alias'); + expect(typeNode).toMatchObject({ kind: 'type_alias', name: 'InternalState', isExported: false, @@ -415,7 +426,7 @@ export const useAuth = () => { expect(varNodes).toHaveLength(0); }); - it('should not extract non-exported const as exported variable', () => { + it('should extract non-exported const as non-exported variable', () => { const code = ` const internalConfig = { debug: true, @@ -423,10 +434,10 @@ const internalConfig = { `; const result = extractFromSource('internal.ts', code); - // Non-exported const should NOT create a variable node - // (only export_statement triggers extractExportedVariables) - const varNodes = result.nodes.filter((n) => n.kind === 'variable' && n.name === 'internalConfig'); - expect(varNodes).toHaveLength(0); + // Non-exported const at file level should be extracted as a constant (not exported) + const varNodes = result.nodes.filter((n) => (n.kind === 'variable' || n.kind === 'constant') && n.name === 'internalConfig'); + expect(varNodes).toHaveLength(1); + expect(varNodes[0]?.isExported).toBeFalsy(); }); it('should extract Zod schema exports', () => { @@ -463,6 +474,93 @@ export const authMachine = createMachine({ }); }); +describe('File Node Extraction', () => { + it('should create a file-kind node for each parsed file', () => { + const code = ` +export function greet(name: string): string { + return "Hello " + name; +} +`; + const result = extractFromSource('greeter.ts', code); + + const fileNode = result.nodes.find((n) => n.kind === 'file'); + expect(fileNode).toBeDefined(); + expect(fileNode?.name).toBe('greeter.ts'); + expect(fileNode?.filePath).toBe('greeter.ts'); + expect(fileNode?.language).toBe('typescript'); + expect(fileNode?.startLine).toBe(1); + }); + + it('should create file nodes for Python files', () => { + const code = ` +def main(): + pass +`; + const result = extractFromSource('main.py', code); + + const fileNode = result.nodes.find((n) => n.kind === 'file'); + expect(fileNode).toBeDefined(); + expect(fileNode?.name).toBe('main.py'); + expect(fileNode?.language).toBe('python'); + }); + + it('should create containment edges from file node to top-level declarations', () => { + const code = ` +export function foo() {} +export function bar() {} +`; + const result = extractFromSource('fns.ts', code); + + const fileNode = result.nodes.find((n) => n.kind === 'file'); + expect(fileNode).toBeDefined(); + + // There should be contains edges from the file node to each function + const containsEdges = result.edges.filter( + (e) => e.source === fileNode?.id && e.kind === 'contains' + ); + expect(containsEdges.length).toBeGreaterThanOrEqual(2); + }); +}); + +describe('Arrow Function Variable Extraction', () => { + it('should extract const arrow functions as function nodes', () => { + const code = ` +const handleClick = () => { + console.log('clicked'); +}; +`; + const result = extractFromSource('handler.ts', code); + + const funcNode = result.nodes.find((n) => n.kind === 'function' && n.name === 'handleClick'); + expect(funcNode).toBeDefined(); + expect(funcNode?.kind).toBe('function'); + }); + + it('should detect async arrow functions', () => { + const code = ` +export const fetchUser = async (id: string) => { + return await db.find(id); +}; +`; + const result = extractFromSource('api.ts', code); + + const funcNode = result.nodes.find((n) => n.kind === 'function' && n.name === 'fetchUser'); + expect(funcNode).toBeDefined(); + expect(funcNode?.isExported).toBe(true); + }); + + it('should not create duplicate nodes for arrow functions in export statements', () => { + const code = ` +export const compute = (x: number) => x * 2; +`; + const result = extractFromSource('math.ts', code); + + const funcNodes = result.nodes.filter((n) => n.kind === 'function' && n.name === 'compute'); + // Should appear only once, not duplicated between extractFunctionVariable and extractFunction + expect(funcNodes).toHaveLength(1); + }); +}); + describe('Python Extraction', () => { it('should extract function definitions', () => { const code = ` @@ -473,8 +571,11 @@ def calculate_total(items: list, tax_rate: float) -> float: `; const result = extractFromSource('calc.py', code); - expect(result.nodes).toHaveLength(1); - expect(result.nodes[0]).toMatchObject({ + const fileNode = result.nodes.find((n) => n.kind === 'file'); + expect(fileNode).toBeDefined(); + + const funcNode = result.nodes.find((n) => n.kind === 'function'); + expect(funcNode).toMatchObject({ kind: 'function', name: 'calculate_total', language: 'python', diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 5ef676b..169eee5 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -156,6 +156,14 @@ function loadGrammar(language: Language): unknown | null { } } +/** + * Get a grammar by language, loading it lazily if needed. + * Exported for direct grammar access without parser initialization. + */ +export function getGrammar(language: string): unknown | null { + return loadGrammar(language as Language); +} + /** * Get a parser for the specified language */ diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 3fa6fc9..325c333 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -18,10 +18,16 @@ import { import { QueryBuilder } from '../db/queries'; import { extractFromSource } from './tree-sitter'; import { detectLanguage, isLanguageSupported } from './grammars'; -import { logDebug } from '../errors'; +import { logDebug, logWarn } from '../errors'; import { captureException } from '../sentry'; import { validatePathWithinRoot } from '../utils'; +/** + * Number of files to read in parallel during indexing. + * File reads are I/O-bound; batching overlaps I/O wait with CPU parse work. + */ +const FILE_IO_BATCH_SIZE = 10; + /** * Progress callback for indexing operations */ @@ -129,22 +135,25 @@ export function scanDirectory( ): string[] { const files: string[] = []; let count = 0; - const visitedRealPaths = new Set(); // Symlink cycle detection + // Track visited real paths to detect symlink cycles + const visitedDirs = new Set(); function walk(dir: string): void { - // Symlink cycle detection: resolve real path and skip if already visited + // Resolve real path to detect symlink cycles + let realDir: string; try { - const realDir = fs.realpathSync(dir); - if (visitedRealPaths.has(realDir)) { - logDebug('Skipping directory to prevent symlink cycle', { dir, realDir }); - return; - } - visitedRealPaths.add(realDir); + realDir = fs.realpathSync(dir); } catch { - // If realpath fails, skip this directory + logDebug('Skipping unresolvable directory', { dir }); return; } + if (visitedDirs.has(realDir)) { + logDebug('Skipping already-visited directory (symlink cycle)', { dir, realDir }); + return; + } + visitedDirs.add(realDir); + // Check for .codegraphignore marker file - skip entire directory tree if present const ignoreMarker = path.join(dir, CODEGRAPH_IGNORE_MARKER); if (fs.existsSync(ignoreMarker)) { @@ -283,10 +292,11 @@ export class ExtractionOrchestrator { }; } - // Phase 2: Parse files + // Phase 2: Parse files (read in parallel batches, parse/store sequentially) const total = files.length; + let processed = 0; - for (let i = 0; i < files.length; i++) { + for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) { if (signal?.aborted) { return { success: false, @@ -299,26 +309,69 @@ export class ExtractionOrchestrator { }; } - const filePath = files[i]!; - onProgress?.({ - phase: 'parsing', - current: i + 1, - total, - currentFile: filePath, - }); + const batch = files.slice(i, i + FILE_IO_BATCH_SIZE); - const result = await this.indexFile(filePath); + // Read files in parallel (with path validation before any I/O) + const fileContents = await Promise.all( + batch.map(async (fp) => { + try { + const fullPath = validatePathWithinRoot(this.rootDir, fp); + if (!fullPath) { + logWarn('Path traversal blocked in batch reader', { filePath: fp }); + return { filePath: fp, content: null as string | null, stats: null as fs.Stats | null, error: new Error('Path traversal blocked') }; + } + const content = await fsp.readFile(fullPath, 'utf-8'); + const stats = await fsp.stat(fullPath); + return { filePath: fp, content, stats, error: null as Error | null }; + } catch (err) { + return { filePath: fp, content: null as string | null, stats: null as fs.Stats | null, error: err as Error }; + } + }) + ); - if (result.errors.length > 0) { - errors.push(...result.errors); - } + // Parse and store sequentially + for (const { filePath, content, stats, error } of fileContents) { + if (signal?.aborted) { + return { + success: false, + filesIndexed, + filesSkipped, + nodesCreated: totalNodes, + edgesCreated: totalEdges, + errors: [{ message: 'Aborted', severity: 'error' }, ...errors], + durationMs: Date.now() - startTime, + }; + } - if (result.nodes.length > 0) { - filesIndexed++; - totalNodes += result.nodes.length; - totalEdges += result.edges.length; - } else if (result.errors.length === 0) { - filesSkipped++; + processed++; + onProgress?.({ + phase: 'parsing', + current: processed, + total, + currentFile: filePath, + }); + + if (error || content === null || stats === null) { + errors.push({ + message: `Failed to read file: ${error instanceof Error ? error.message : String(error)}`, + severity: 'error', + }); + continue; + } + + const result = await this.indexFileWithContent(filePath, content, stats); + + if (result.errors.length > 0) { + errors.push(...result.errors); + } + + if (result.nodes.length > 0) { + filesIndexed++; + totalNodes += result.nodes.length; + totalEdges += result.edges.length; + } else if (result.errors.length === 0) { + filesSkipped++; + } } } @@ -457,6 +510,67 @@ export class ExtractionOrchestrator { return result; } + /** + * Index a single file with pre-read content and stats. + * Used by the parallel batch reader to avoid redundant file I/O. + */ + async indexFileWithContent( + relativePath: string, + content: string, + stats: fs.Stats + ): Promise { + // Prevent path traversal + const fullPath = validatePathWithinRoot(this.rootDir, relativePath); + if (!fullPath) { + logWarn('Path traversal blocked in indexFileWithContent', { relativePath }); + return { + nodes: [], + edges: [], + unresolvedReferences: [], + errors: [{ message: 'Path traversal blocked', severity: 'error' }], + durationMs: 0, + }; + } + + // Check file size + if (stats.size > this.config.maxFileSize) { + return { + nodes: [], + edges: [], + unresolvedReferences: [], + errors: [ + { + message: `File exceeds max size (${stats.size} > ${this.config.maxFileSize})`, + severity: 'warning', + }, + ], + durationMs: 0, + }; + } + + // Detect language + const language = detectLanguage(relativePath); + if (!isLanguageSupported(language)) { + return { + nodes: [], + edges: [], + unresolvedReferences: [], + errors: [], + durationMs: 0, + }; + } + + // Extract from source + const result = extractFromSource(relativePath, content, language); + + // Store in database + if (result.nodes.length > 0 || result.errors.length === 0) { + this.storeExtractionResult(relativePath, content, language, stats, result); + } + + return result; + } + /** * Store extraction result in database */ diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index c04d0c7..ad63460 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -6,6 +6,7 @@ import { SyntaxNode, Tree } from 'tree-sitter'; import * as crypto from 'crypto'; +import * as path from 'path'; import { Language, Node, @@ -875,7 +876,28 @@ export class TreeSitterExtractor { try { this.tree = parser.parse(this.source); + + // Create file node representing the source file + const fileNode: Node = { + id: `file:${this.filePath}`, + kind: 'file', + name: path.basename(this.filePath), + qualifiedName: this.filePath, + filePath: this.filePath, + language: this.language, + startLine: 1, + endLine: this.source.split('\n').length, + startColumn: 0, + endColumn: 0, + isExported: false, + updatedAt: Date.now(), + }; + this.nodes.push(fileNode); + + // Push file node onto stack so top-level declarations get contains edges + this.nodeStack.push(fileNode.id); this.visitNode(this.tree.rootNode); + this.nodeStack.pop(); } catch (error) { captureException(error, { operation: 'tree-sitter-parse', filePath: this.filePath, language: this.language }); this.errors.push({ @@ -905,7 +927,7 @@ export class TreeSitterExtractor { // Check for function declarations // For Python/Ruby, function_definition inside a class should be treated as method if (this.extractor.functionTypes.includes(nodeType)) { - if (this.nodeStack.length > 0 && this.extractor.methodTypes.includes(nodeType)) { + if (this.isInsideClassLikeNode() && this.extractor.methodTypes.includes(nodeType)) { // Inside a class - treat as method this.extractMethod(node); skipChildren = true; // extractMethod visits children via visitFunctionBody @@ -956,9 +978,17 @@ export class TreeSitterExtractor { else if (this.extractor.typeAliasTypes.includes(nodeType)) { this.extractTypeAlias(node); } + // Check for arrow functions / function expressions assigned to variables (JS/TS) + else if (nodeType === 'variable_declarator') { + const valueNode = getChildByField(node, 'value'); + if (valueNode && (valueNode.type === 'arrow_function' || valueNode.type === 'function_expression')) { + this.extractFunctionVariable(node); + skipChildren = true; + } + } // Check for variable declarations (const, let, var, etc.) // Only extract top-level variables (not inside functions/methods) - else if (this.extractor.variableTypes.includes(nodeType) && this.nodeStack.length === 0) { + else if (this.extractor.variableTypes.includes(nodeType) && !this.isInsideClassLikeNode()) { this.extractVariable(node); skipChildren = true; // extractVariable handles children } @@ -1060,6 +1090,81 @@ export class TreeSitterExtractor { return false; } + /** + * Check if the current node stack indicates we are inside a class-like node + * (class, struct, interface, trait). File nodes do not count as class-like. + */ + private isInsideClassLikeNode(): boolean { + if (this.nodeStack.length === 0) return false; + const parentId = this.nodeStack[this.nodeStack.length - 1]; + if (!parentId) return false; + const parentNode = this.nodes.find((n) => n.id === parentId); + if (!parentNode) return false; + return ( + parentNode.kind === 'class' || + parentNode.kind === 'struct' || + parentNode.kind === 'interface' || + parentNode.kind === 'trait' || + parentNode.kind === 'enum' + ); + } + + /** + * Extract an arrow function or function expression assigned to a variable. + * Handles patterns like: const foo = () => {} or const bar = function() {} + */ + private extractFunctionVariable(node: SyntaxNode): void { + if (!this.extractor) return; + + // Only handle variable_declarator where value is arrow_function or function_expression + if (node.type !== 'variable_declarator') return; + + const nameNode = getChildByField(node, 'name'); + const valueNode = getChildByField(node, 'value'); + + if (!nameNode || !valueNode) return; + if (valueNode.type !== 'arrow_function' && valueNode.type !== 'function_expression') return; + + const name = getNodeText(nameNode, this.source); + if (!name) return; + + // Check if exported by walking parents + let isExported = false; + let current = node.parent; + while (current) { + if (current.type === 'export_statement') { + isExported = true; + break; + } + if (current.type === 'program' || current.type === 'module') break; + current = current.parent; + } + + // Build signature from the arrow function parameters + let signature: string | undefined; + const params = getChildByField(valueNode, 'parameters'); + if (params) { + signature = `${name}${getNodeText(params, this.source)}`; + } + + // Check if async + const isAsync = this.extractor.isAsync?.(valueNode); + + const funcNode = this.createNode('function', name, node, { + isExported, + signature: signature || undefined, + isAsync, + }); + + // Push to stack and visit body for call extraction + this.nodeStack.push(funcNode.id); + const body = getChildByField(valueNode, this.extractor.bodyField); + if (body) { + this.visitFunctionBody(body, funcNode.id); + } + this.nodeStack.pop(); + } + /** * Extract a function */ @@ -1160,10 +1265,10 @@ export class TreeSitterExtractor { private extractMethod(node: SyntaxNode): void { if (!this.extractor) return; - // For most languages, only extract as method if inside a class + // For most languages, only extract as method if inside a class-like node // But Go methods are top-level with a receiver, so always treat them as methods - if (this.nodeStack.length === 0 && this.language !== 'go') { - // Top-level and not Go, treat as function + if (!this.isInsideClassLikeNode() && this.language !== 'go') { + // Not inside a class-like node and not Go, treat as function this.extractFunction(node); return; } From 36af284e1039def2738d741907cdae7108e56695 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 10 Feb 2026 15:55:43 -0600 Subject: [PATCH 4/5] Remove redundant code and deduplicate indexFile - Remove extractFunctionVariable() and its dispatch (already handled by extractVariable) - Remove dead getGrammar() export (zero callers) - Deduplicate indexFile by delegating to indexFileWithContent - Remove redundant arrow function variable extraction tests (covered by existing suite) --- __tests__/extraction.test.ts | 39 --------------------- src/extraction/grammars.ts | 8 ----- src/extraction/index.ts | 40 ++-------------------- src/extraction/tree-sitter.ts | 64 ----------------------------------- 4 files changed, 2 insertions(+), 149 deletions(-) diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index a40c4ae..0a5c46b 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -524,45 +524,6 @@ export function bar() {} }); }); -describe('Arrow Function Variable Extraction', () => { - it('should extract const arrow functions as function nodes', () => { - const code = ` -const handleClick = () => { - console.log('clicked'); -}; -`; - const result = extractFromSource('handler.ts', code); - - const funcNode = result.nodes.find((n) => n.kind === 'function' && n.name === 'handleClick'); - expect(funcNode).toBeDefined(); - expect(funcNode?.kind).toBe('function'); - }); - - it('should detect async arrow functions', () => { - const code = ` -export const fetchUser = async (id: string) => { - return await db.find(id); -}; -`; - const result = extractFromSource('api.ts', code); - - const funcNode = result.nodes.find((n) => n.kind === 'function' && n.name === 'fetchUser'); - expect(funcNode).toBeDefined(); - expect(funcNode?.isExported).toBe(true); - }); - - it('should not create duplicate nodes for arrow functions in export statements', () => { - const code = ` -export const compute = (x: number) => x * 2; -`; - const result = extractFromSource('math.ts', code); - - const funcNodes = result.nodes.filter((n) => n.kind === 'function' && n.name === 'compute'); - // Should appear only once, not duplicated between extractFunctionVariable and extractFunction - expect(funcNodes).toHaveLength(1); - }); -}); - describe('Python Extraction', () => { it('should extract function definitions', () => { const code = ` diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 169eee5..5ef676b 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -156,14 +156,6 @@ function loadGrammar(language: Language): unknown | null { } } -/** - * Get a grammar by language, loading it lazily if needed. - * Exported for direct grammar access without parser initialization. - */ -export function getGrammar(language: string): unknown | null { - return loadGrammar(language as Language); -} - /** * Get a parser for the specified language */ diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 4a1a573..8b74a98 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -487,7 +487,7 @@ export class ExtractionOrchestrator { }; } - // Check file exists and is readable + // Read file content and stats let content: string; let stats: fs.Stats; try { @@ -509,43 +509,7 @@ export class ExtractionOrchestrator { }; } - // Check file size - if (stats.size > this.config.maxFileSize) { - return { - nodes: [], - edges: [], - unresolvedReferences: [], - errors: [ - { - message: `File exceeds max size (${stats.size} > ${this.config.maxFileSize})`, - severity: 'warning', - }, - ], - durationMs: 0, - }; - } - - // Detect language - const language = detectLanguage(relativePath); - if (!isLanguageSupported(language)) { - return { - nodes: [], - edges: [], - unresolvedReferences: [], - errors: [], - durationMs: 0, - }; - } - - // Extract from source - const result = extractFromSource(relativePath, content, language); - - // Store in database - if (result.nodes.length > 0 || result.errors.length === 0) { - this.storeExtractionResult(relativePath, content, language, stats, result); - } - - return result; + return this.indexFileWithContent(relativePath, content, stats); } /** diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index ad63460..b19b7a7 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -978,14 +978,6 @@ export class TreeSitterExtractor { else if (this.extractor.typeAliasTypes.includes(nodeType)) { this.extractTypeAlias(node); } - // Check for arrow functions / function expressions assigned to variables (JS/TS) - else if (nodeType === 'variable_declarator') { - const valueNode = getChildByField(node, 'value'); - if (valueNode && (valueNode.type === 'arrow_function' || valueNode.type === 'function_expression')) { - this.extractFunctionVariable(node); - skipChildren = true; - } - } // Check for variable declarations (const, let, var, etc.) // Only extract top-level variables (not inside functions/methods) else if (this.extractor.variableTypes.includes(nodeType) && !this.isInsideClassLikeNode()) { @@ -1109,62 +1101,6 @@ export class TreeSitterExtractor { ); } - /** - * Extract an arrow function or function expression assigned to a variable. - * Handles patterns like: const foo = () => {} or const bar = function() {} - */ - private extractFunctionVariable(node: SyntaxNode): void { - if (!this.extractor) return; - - // Only handle variable_declarator where value is arrow_function or function_expression - if (node.type !== 'variable_declarator') return; - - const nameNode = getChildByField(node, 'name'); - const valueNode = getChildByField(node, 'value'); - - if (!nameNode || !valueNode) return; - if (valueNode.type !== 'arrow_function' && valueNode.type !== 'function_expression') return; - - const name = getNodeText(nameNode, this.source); - if (!name) return; - - // Check if exported by walking parents - let isExported = false; - let current = node.parent; - while (current) { - if (current.type === 'export_statement') { - isExported = true; - break; - } - if (current.type === 'program' || current.type === 'module') break; - current = current.parent; - } - - // Build signature from the arrow function parameters - let signature: string | undefined; - const params = getChildByField(valueNode, 'parameters'); - if (params) { - signature = `${name}${getNodeText(params, this.source)}`; - } - - // Check if async - const isAsync = this.extractor.isAsync?.(valueNode); - - const funcNode = this.createNode('function', name, node, { - isExported, - signature: signature || undefined, - isAsync, - }); - - // Push to stack and visit body for call extraction - this.nodeStack.push(funcNode.id); - const body = getChildByField(valueNode, this.extractor.bodyField); - if (body) { - this.visitFunctionBody(body, funcNode.id); - } - this.nodeStack.pop(); - } - /** * Extract a function */ From a15ad692887de6a79934060acf639569b0d0978b Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 10 Feb 2026 16:05:46 -0600 Subject: [PATCH 5/5] Remove unused detectApiIntent and inferRouteDirectories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both functions have zero callers — dead code on arrival. Remove them and their tests (9 tests) to keep the module focused on what's actually used: search term extraction, path relevance scoring, and kind bonuses. --- __tests__/search.test.ts | 81 +-------------------------------------- src/search/query-utils.ts | 42 +------------------- 2 files changed, 2 insertions(+), 121 deletions(-) diff --git a/__tests__/search.test.ts b/__tests__/search.test.ts index 25b7c26..e8e1bc3 100644 --- a/__tests__/search.test.ts +++ b/__tests__/search.test.ts @@ -1,7 +1,7 @@ /** * Search Query Utilities Tests * - * Tests multi-signal scoring, kind bonuses, path relevance, and API intent detection. + * Tests multi-signal scoring, kind bonuses, and path relevance. */ import { describe, it, expect } from 'vitest'; @@ -9,8 +9,6 @@ import { extractSearchTerms, scorePathRelevance, kindBonus, - detectApiIntent, - inferRouteDirectories, STOP_WORDS, } from '../src/search/query-utils'; @@ -114,83 +112,6 @@ describe('Search Query Utilities', () => { }); }); - describe('detectApiIntent', () => { - it('should detect API-related queries', () => { - expect(detectApiIntent('find the API endpoint for users')).toBe(true); - expect(detectApiIntent('where is the login route')).toBe(true); - expect(detectApiIntent('show me the request handler')).toBe(true); - }); - - it('should detect HTTP method patterns', () => { - expect(detectApiIntent('GET /api/users')).toBe(true); - expect(detectApiIntent('post /users/create')).toBe(true); - }); - - it('should detect REST and GraphQL', () => { - expect(detectApiIntent('REST API for payments')).toBe(true); - expect(detectApiIntent('GraphQL resolver for orders')).toBe(true); - }); - - it('should not detect non-API queries', () => { - expect(detectApiIntent('fix the login bug')).toBe(false); - expect(detectApiIntent('add dark mode support')).toBe(false); - }); - - it('should detect controller and middleware mentions', () => { - expect(detectApiIntent('find the auth controller')).toBe(true); - expect(detectApiIntent('CORS middleware configuration')).toBe(true); - }); - }); - - describe('inferRouteDirectories', () => { - it('should detect route directories', () => { - const files = [ - 'src/routes/auth.ts', - 'src/routes/users.ts', - 'src/utils/format.ts', - ]; - const dirs = inferRouteDirectories(files); - expect(dirs).toBeDefined(); - if (dirs) { - expect(dirs.some(d => d.includes('route'))).toBe(true); - } - }); - - it('should detect controller directories', () => { - const files = [ - 'src/controllers/AuthController.ts', - 'src/models/User.ts', - ]; - const dirs = inferRouteDirectories(files); - expect(dirs).toBeDefined(); - if (dirs) { - expect(dirs.some(d => d.includes('controller'))).toBe(true); - } - }); - - it('should detect api directories', () => { - const files = [ - 'src/api/v1/users.ts', - 'src/api/v1/orders.ts', - ]; - const dirs = inferRouteDirectories(files); - expect(dirs).toBeDefined(); - if (dirs) { - expect(dirs.some(d => d.includes('api'))).toBe(true); - } - }); - - it('should return undefined when no route dirs found', () => { - const files = [ - 'src/utils/format.ts', - 'src/models/User.ts', - 'src/index.ts', - ]; - const dirs = inferRouteDirectories(files); - expect(dirs).toBeUndefined(); - }); - }); - describe('STOP_WORDS', () => { it('should contain common English stop words', () => { expect(STOP_WORDS.has('the')).toBe(true); diff --git a/src/search/query-utils.ts b/src/search/query-utils.ts index e08d66e..8f16340 100644 --- a/src/search/query-utils.ts +++ b/src/search/query-utils.ts @@ -1,7 +1,7 @@ /** * Search Query Utilities * - * Shared module for search term extraction, scoring, and intent detection. + * Shared module for search term extraction and scoring. */ import * as path from 'path'; @@ -87,43 +87,3 @@ export function kindBonus(kind: Node['kind']): number { }; return bonuses[kind] ?? 0; } - -/** - * Detect if a query has API/endpoint intent - */ -export function detectApiIntent(query: string): boolean { - const apiPatterns = [ - /\bapi\b/i, /\bendpoint/i, /\broute/i, /\bhandler/i, - /\bcontroller/i, /\bmiddleware/i, /\brest\b/i, /\bgraphql/i, - /\bget\s+\//, /\bpost\s+\//, /\bput\s+\//, /\bdelete\s+\//, - /\brequest/i, /\bresponse/i, /\bhttp/i, - ]; - return apiPatterns.some(p => p.test(query)); -} - -/** - * Infer route/controller directories from project structure - * Returns undefined if no route directories are detected - */ -export function inferRouteDirectories(files: string[]): string[] | undefined { - const routeDirs = new Set(); - const routePatterns = [ - /routes?\//i, /controllers?\//i, /handlers?\//i, - /api\//i, /endpoints?\//i, - ]; - - for (const file of files) { - for (const pattern of routePatterns) { - if (pattern.test(file)) { - const match = file.match(pattern); - if (match) { - const idx = file.indexOf(match[0]); - const dir = file.substring(0, idx + match[0].length - 1); - routeDirs.add(dir); - } - } - } - } - - return routeDirs.size > 0 ? Array.from(routeDirs) : undefined; -}