diff --git a/__tests__/evaluation/runner.ts b/__tests__/evaluation/runner.ts new file mode 100644 index 0000000..7ff0435 --- /dev/null +++ b/__tests__/evaluation/runner.ts @@ -0,0 +1,123 @@ +import { execSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { CodeGraph } from '../../src/index.js'; +import { scoreSearchNodes, scoreFindRelevantContext } from './scoring.js'; +import { testCases } from './test-cases.js'; +import type { EvalReport, EvalResult } from './types.js'; + +const codebasePath = process.env.EVAL_CODEBASE || process.argv[2]; +if (!codebasePath) { + console.error('Usage: EVAL_CODEBASE=/path/to/codebase npx tsx __tests__/evaluation/runner.ts'); + console.error(' or: npx tsx __tests__/evaluation/runner.ts /path/to/codebase'); + process.exit(1); +} + +const resolvedPath = path.resolve(codebasePath); +if (!fs.existsSync(path.join(resolvedPath, '.codegraph', 'codegraph.db'))) { + console.error(`No .codegraph/codegraph.db found at ${resolvedPath}`); + process.exit(1); +} + +let codegraphSha = 'unknown'; +try { + codegraphSha = execSync('git rev-parse --short HEAD', { encoding: 'utf-8' }).trim(); +} catch {} + +console.log(`\nCodeGraph Eval — ${path.basename(resolvedPath)}`); +console.log(`Codebase: ${resolvedPath}`); +console.log(`Commit: ${codegraphSha}`); +console.log(`Cases: ${testCases.length}`); +console.log(''); + +async function run() { + const cg = CodeGraph.openSync(resolvedPath); + const results: EvalResult[] = []; + + for (const tc of testCases) { + const start = performance.now(); + + if (tc.api === 'searchNodes') { + const searchResults = cg.searchNodes(tc.query, { + limit: 10, + kinds: tc.kinds, + ...(tc.options as Record), + }); + const latency = performance.now() - start; + const result = scoreSearchNodes(tc.id, tc.expectedSymbols, searchResults, latency); + results.push(result); + } else { + const subgraph = await cg.findRelevantContext(tc.query, { + searchLimit: 8, + traversalDepth: 3, + maxNodes: 80, + minScore: 0.2, + ...(tc.options as Record), + }); + const latency = performance.now() - start; + const result = scoreFindRelevantContext(tc.id, tc.expectedSymbols, subgraph, latency); + results.push(result); + } + } + + cg.close(); + + // Print results table + const maxIdLen = Math.max(...results.map((r) => r.caseId.length)); + + for (const r of results) { + const status = r.pass ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m'; + const id = r.caseId.padEnd(maxIdLen); + const recall = `recall=${r.recall.toFixed(2)}`; + const extra = + r.edgeDensity !== undefined + ? `density=${r.edgeDensity.toFixed(2)}` + : `mrr=${r.mrr.toFixed(2)}`; + const latency = `${Math.round(r.latencyMs)}ms`; + + console.log(` ${id} ${status} ${recall} ${extra} ${latency}`); + + if (r.missedSymbols.length > 0) { + console.log(` ${' '.repeat(maxIdLen)} missed: ${r.missedSymbols.join(', ')}`); + } + } + + // Summary + const passed = results.filter((r) => r.pass).length; + const failed = results.length - passed; + const meanRecall = results.reduce((s, r) => s + r.recall, 0) / results.length; + const mrrResults = results.filter((r) => r.mrr > 0 || r.caseId.startsWith('search-')); + const meanMRR = + mrrResults.length > 0 ? mrrResults.reduce((s, r) => s + r.mrr, 0) / mrrResults.length : 0; + + console.log(''); + const summaryColor = failed === 0 ? '\x1b[32m' : '\x1b[33m'; + console.log( + `${summaryColor}SUMMARY: ${passed}/${results.length} passed | recall=${meanRecall.toFixed(2)} | mrr=${meanMRR.toFixed(2)}\x1b[0m` + ); + + // Save JSON report + const report: EvalReport = { + timestamp: new Date().toISOString(), + codebasePath: resolvedPath, + codegraphSha, + summary: { total: results.length, passed, failed, meanRecall, meanMRR }, + results, + }; + + const resultsDir = path.join(__dirname, 'results'); + fs.mkdirSync(resultsDir, { recursive: true }); + const reportFile = path.join( + resultsDir, + `${new Date().toISOString().replace(/[:.]/g, '-')}.json` + ); + fs.writeFileSync(reportFile, JSON.stringify(report, null, 2)); + console.log(`\nReport saved: ${reportFile}`); + + process.exit(failed > 0 ? 1 : 0); +} + +run().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/__tests__/evaluation/scoring.ts b/__tests__/evaluation/scoring.ts new file mode 100644 index 0000000..b20f604 --- /dev/null +++ b/__tests__/evaluation/scoring.ts @@ -0,0 +1,82 @@ +import type { EvalResult } from './types.js'; + +export const PASS_THRESHOLD = 0.5; + +export function scoreSearchNodes( + caseId: string, + expectedSymbols: string[], + results: Array<{ node: { name: string }; score: number }>, + latencyMs: number +): EvalResult { + const expectedLower = expectedSymbols.map((s) => s.toLowerCase()); + const resultNames = results.map((r) => r.node.name.toLowerCase()); + + const found: string[] = []; + const missed: string[] = []; + let firstRank = 0; + + for (let i = 0; i < expectedLower.length; i++) { + const idx = resultNames.indexOf(expectedLower[i]); + if (idx !== -1) { + found.push(expectedSymbols[i]); + if (firstRank === 0) firstRank = idx + 1; + } else { + missed.push(expectedSymbols[i]); + } + } + + const recall = expectedSymbols.length > 0 ? found.length / expectedSymbols.length : 0; + const mrr = firstRank > 0 ? 1 / firstRank : 0; + + return { + caseId, + pass: recall >= PASS_THRESHOLD, + recall, + mrr, + foundSymbols: found, + missedSymbols: missed, + latencyMs, + }; +} + +export function scoreFindRelevantContext( + caseId: string, + expectedSymbols: string[], + subgraph: { nodes: Map; edges: unknown[]; roots: string[] }, + latencyMs: number +): EvalResult { + const expectedLower = new Set(expectedSymbols.map((s) => s.toLowerCase())); + const nodeNames = new Set(); + for (const node of subgraph.nodes.values()) { + nodeNames.add(node.name.toLowerCase()); + } + + const found: string[] = []; + const missed: string[] = []; + + for (const sym of expectedSymbols) { + if (nodeNames.has(sym.toLowerCase())) { + found.push(sym); + } else { + missed.push(sym); + } + } + + const recall = expectedSymbols.length > 0 ? found.length / expectedSymbols.length : 0; + const nodeCount = subgraph.nodes.size; + const edgeCount = subgraph.edges.length; + const edgeDensity = nodeCount > 0 ? edgeCount / nodeCount : 0; + + return { + caseId, + pass: recall >= PASS_THRESHOLD, + recall, + mrr: 0, + foundSymbols: found, + missedSymbols: missed, + nodeCount, + edgeCount, + edgeDensity, + latencyMs, + }; +} diff --git a/__tests__/evaluation/test-cases.ts b/__tests__/evaluation/test-cases.ts new file mode 100644 index 0000000..db478fb --- /dev/null +++ b/__tests__/evaluation/test-cases.ts @@ -0,0 +1,93 @@ +import type { EvalTestCase } from './types.js'; + +export const testCases: EvalTestCase[] = [ + // === searchNodes: Symbol Lookup Precision === + + { + id: 'search-class-exact', + query: 'TransportService', + api: 'searchNodes', + expectedSymbols: ['TransportService'], + kinds: ['class'], + }, + { + id: 'search-method-qualified', + query: 'TransportService sendRequest', + api: 'searchNodes', + expectedSymbols: ['sendRequest'], + kinds: ['method'], + }, + { + id: 'search-interface', + query: 'ActionListener', + api: 'searchNodes', + expectedSymbols: ['ActionListener'], + kinds: ['interface'], + }, + { + id: 'search-enum', + query: 'RestStatus', + api: 'searchNodes', + expectedSymbols: ['RestStatus'], + kinds: ['enum'], + }, + { + id: 'search-exception', + query: 'SearchPhaseExecutionException', + api: 'searchNodes', + expectedSymbols: ['SearchPhaseExecutionException'], + kinds: ['class'], + }, + { + id: 'search-nested-class', + query: 'Engine Index', + api: 'searchNodes', + expectedSymbols: ['Index'], + kinds: ['class'], + }, + + // === findRelevantContext: Exploration Quality === + + { + id: 'explore-rest-layer', + query: 'How does the REST layer handle HTTP requests?', + api: 'findRelevantContext', + expectedSymbols: ['RestController', 'RestHandler', 'BaseRestHandler', 'RestRequest'], + options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 }, + }, + { + id: 'explore-search-execution', + query: 'How does search execution work from request to shard?', + api: 'findRelevantContext', + expectedSymbols: ['TransportSearchAction', 'AbstractSearchAsyncAction', 'QueryPhase', 'FetchPhase'], + options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 }, + }, + { + id: 'explore-bulk-indexing', + query: 'How does bulk indexing work?', + api: 'findRelevantContext', + expectedSymbols: ['TransportBulkAction', 'BulkRequest', 'BulkResponse'], + options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 }, + }, + { + id: 'explore-shard-allocation', + query: 'How does shard rebalancing and allocation work?', + api: 'findRelevantContext', + expectedSymbols: ['AllocationService', 'BalancedShardsAllocator'], + options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 }, + }, + { + id: 'explore-transport-search', + query: 'How does TransportService connect to SearchTransportService?', + api: 'findRelevantContext', + expectedSymbols: ['TransportService', 'SearchTransportService'], + options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 }, + }, + { + id: 'explore-engine-implementations', + query: 'What are the Engine implementations for indexing?', + api: 'findRelevantContext', + expectedSymbols: ['InternalEngine', 'ReadOnlyEngine', 'Engine'], + options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 }, + }, +]; diff --git a/__tests__/evaluation/types.ts b/__tests__/evaluation/types.ts new file mode 100644 index 0000000..64a2427 --- /dev/null +++ b/__tests__/evaluation/types.ts @@ -0,0 +1,37 @@ +import type { NodeKind } from '../../src/types.js'; + +export interface EvalTestCase { + id: string; + query: string; + api: 'searchNodes' | 'findRelevantContext'; + expectedSymbols: string[]; + kinds?: NodeKind[]; + options?: Record; +} + +export interface EvalResult { + caseId: string; + pass: boolean; + recall: number; + mrr: number; + foundSymbols: string[]; + missedSymbols: string[]; + nodeCount?: number; + edgeCount?: number; + edgeDensity?: number; + latencyMs: number; +} + +export interface EvalReport { + timestamp: string; + codebasePath: string; + codegraphSha: string; + summary: { + total: number; + passed: number; + failed: number; + meanRecall: number; + meanMRR: number; + }; + results: EvalResult[]; +} diff --git a/src/context/index.ts b/src/context/index.ts index cf5aedc..ddb022d 100644 --- a/src/context/index.ts +++ b/src/context/index.ts @@ -68,6 +68,14 @@ function extractSymbolsFromQuery(query: string): string[] { } } + // Extract ALL_CAPS acronyms (2+ chars, e.g., REST, HTTP, LRU, API) + const acronymPattern = /\b([A-Z]{2,})\b/g; + while ((match = acronymPattern.exec(query)) !== null) { + if (match[1]) { + symbols.add(match[1]); + } + } + // Extract dot.notation and split into parts (e.g., "app.isPackaged" -> ["app", "isPackaged"]) const dotPattern = /\b([a-zA-Z][a-zA-Z0-9]*(?:\.[a-zA-Z][a-zA-Z0-9]*)+)\b/g; while ((match = dotPattern.exec(query)) !== null) { @@ -107,6 +115,19 @@ function extractSymbolsFromQuery(query: string): string[] { 'more', 'most', 'very', 'being', 'having', 'doing', 'system', 'need', 'needs', 'want', 'wants', 'like', 'look', 'change', 'changes', 'changed', 'changing', + // Common English nouns/verbs that match thousands of unrelated code symbols + 'layer', 'handle', 'handles', 'handling', 'incoming', 'outgoing', + 'data', 'flow', 'flows', 'level', 'levels', 'request', 'requests', + 'response', 'responses', 'implement', 'implements', 'implementation', + 'interface', 'interfaces', 'class', 'classes', 'method', 'methods', + 'trigger', 'triggers', 'affected', 'affect', 'affects', + 'else', 'code', 'failing', 'failed', 'silently', 'decide', 'decides', + 'connect', 'connection', 'connections', + 'return', 'returns', 'returned', 'take', 'takes', 'taken', + 'send', 'sends', 'receive', 'receives', 'process', 'processes', + 'check', 'checks', 'checked', 'create', 'creates', 'created', + 'read', 'reads', 'write', 'writes', 'written', + 'start', 'starts', 'stop', 'stops', 'run', 'runs', 'running', ]); return Array.from(symbols).filter(s => !commonWords.has(s.toLowerCase())); @@ -327,6 +348,43 @@ export class ContextBuilder { } } + // Step 2b: Search for extracted symbols as definition (class/interface) prefixes. + // When the user writes "REST", "bulk", or "allocation", they usually mean classes + // like RestController, BulkRequest, AllocationService — not nodes named exactly that. + if (symbolsFromQuery.length > 0) { + const definitionKinds: NodeKind[] = ['class', 'interface', 'struct', 'trait', + 'protocol', 'enum', 'type_alias']; + for (const sym of symbolsFromQuery) { + // Title-case the symbol: "REST" → "Rest", "bulk" → "Bulk", "allocation" → "Allocation" + const titleCased = sym.charAt(0).toUpperCase() + sym.slice(1).toLowerCase(); + if (titleCased === sym) continue; // already title-case (e.g., "Engine") — handled by exact match + // Fetch more results since popular prefixes have many matches + const prefixResults = this.queries.searchNodes(titleCased, { + limit: 30, + kinds: definitionKinds, + }); + const matched: SearchResult[] = []; + for (const r of prefixResults) { + if (r.node.name.toLowerCase().startsWith(titleCased.toLowerCase())) { + // Favor shorter names: "AllocationService" (18 chars) over + // "AllocationBalancingRoundMetrics" (31 chars). Core classes tend + // to have concise names; test/helper classes are verbose. + const brevityBonus = Math.max(0, 10 - (r.node.name.length - titleCased.length) / 3); + matched.push({ ...r, score: r.score + 15 + brevityBonus }); + } + } + matched.sort((a, b) => b.score - a.score); + for (const r of matched.slice(0, Math.ceil(opts.searchLimit))) { + const existing = exactMatches.find(e => e.node.id === r.node.id); + if (!existing) { + exactMatches.push(r); + } + } + } + exactMatches.sort((a, b) => b.score - a.score); + exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 3)); + } + // Step 3: Try semantic search if vector manager is available let semanticResults: SearchResult[] = []; if (this.vectorManager && this.vectorManager.isInitialized()) { @@ -352,10 +410,19 @@ export class ContextBuilder { // Search each term individually to get broader coverage, // then boost results that match multiple terms const termResultsMap = new Map(); + // When no explicit kind filter is set, exclude imports — they flood FTS + // results with qualified name matches (e.g., "REST" matches 445K import paths) + // but are almost never what exploration queries want. + const searchKinds = opts.nodeKinds && opts.nodeKinds.length > 0 + ? opts.nodeKinds + : ['file', 'module', 'class', 'struct', 'interface', 'trait', 'protocol', + 'function', 'method', 'property', 'field', 'variable', 'constant', + 'enum', 'enum_member', 'type_alias', 'namespace', 'export', + 'route', 'component'] as NodeKind[]; for (const term of searchTerms) { const termResults = this.queries.searchNodes(term, { limit: opts.searchLimit * 2, - kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, + kinds: searchKinds, }); for (const r of termResults) { const existing = termResultsMap.get(r.node.id);