feat: Add comprehensive evaluation framework for CodeGraph API testing

Introduces automated testing infrastructure to measure CodeGraph performance across searchNodes and findRelevantContext APIs. Includes recall/MRR scoring metrics, predefined test cases for symbol lookup and context exploration, and JSON report generation. Enhances context building with acronym extraction, definition prefix matching, and improved FTS filtering to exclude imports by default.
2026-04-06 13:24:27 -05:00
parent d4258b1651
commit 13d3ff3613
5 changed files with 403 additions and 1 deletions
@@ -0,0 +1,123 @@
+import { execSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import { CodeGraph } from '../../src/index.js';
+import { scoreSearchNodes, scoreFindRelevantContext } from './scoring.js';
+import { testCases } from './test-cases.js';
+import type { EvalReport, EvalResult } from './types.js';
+
+const codebasePath = process.env.EVAL_CODEBASE || process.argv[2];
+if (!codebasePath) {
+  console.error('Usage: EVAL_CODEBASE=/path/to/codebase npx tsx __tests__/evaluation/runner.ts');
+  console.error('   or: npx tsx __tests__/evaluation/runner.ts /path/to/codebase');
+  process.exit(1);
+}
+
+const resolvedPath = path.resolve(codebasePath);
+if (!fs.existsSync(path.join(resolvedPath, '.codegraph', 'codegraph.db'))) {
+  console.error(`No .codegraph/codegraph.db found at ${resolvedPath}`);
+  process.exit(1);
+}
+
+let codegraphSha = 'unknown';
+try {
+  codegraphSha = execSync('git rev-parse --short HEAD', { encoding: 'utf-8' }).trim();
+} catch {}
+
+console.log(`\nCodeGraph Eval — ${path.basename(resolvedPath)}`);
+console.log(`Codebase: ${resolvedPath}`);
+console.log(`Commit:   ${codegraphSha}`);
+console.log(`Cases:    ${testCases.length}`);
+console.log('');
+
+async function run() {
+  const cg = CodeGraph.openSync(resolvedPath);
+  const results: EvalResult[] = [];
+
+  for (const tc of testCases) {
+    const start = performance.now();
+
+    if (tc.api === 'searchNodes') {
+      const searchResults = cg.searchNodes(tc.query, {
+        limit: 10,
+        kinds: tc.kinds,
+        ...(tc.options as Record<string, unknown>),
+      });
+      const latency = performance.now() - start;
+      const result = scoreSearchNodes(tc.id, tc.expectedSymbols, searchResults, latency);
+      results.push(result);
+    } else {
+      const subgraph = await cg.findRelevantContext(tc.query, {
+        searchLimit: 8,
+        traversalDepth: 3,
+        maxNodes: 80,
+        minScore: 0.2,
+        ...(tc.options as Record<string, unknown>),
+      });
+      const latency = performance.now() - start;
+      const result = scoreFindRelevantContext(tc.id, tc.expectedSymbols, subgraph, latency);
+      results.push(result);
+    }
+  }
+
+  cg.close();
+
+  // Print results table
+  const maxIdLen = Math.max(...results.map((r) => r.caseId.length));
+
+  for (const r of results) {
+    const status = r.pass ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m';
+    const id = r.caseId.padEnd(maxIdLen);
+    const recall = `recall=${r.recall.toFixed(2)}`;
+    const extra =
+      r.edgeDensity !== undefined
+        ? `density=${r.edgeDensity.toFixed(2)}`
+        : `mrr=${r.mrr.toFixed(2)}`;
+    const latency = `${Math.round(r.latencyMs)}ms`;
+
+    console.log(`  ${id}  ${status}  ${recall}  ${extra}  ${latency}`);
+
+    if (r.missedSymbols.length > 0) {
+      console.log(`  ${' '.repeat(maxIdLen)}        missed: ${r.missedSymbols.join(', ')}`);
+    }
+  }
+
+  // Summary
+  const passed = results.filter((r) => r.pass).length;
+  const failed = results.length - passed;
+  const meanRecall = results.reduce((s, r) => s + r.recall, 0) / results.length;
+  const mrrResults = results.filter((r) => r.mrr > 0 || r.caseId.startsWith('search-'));
+  const meanMRR =
+    mrrResults.length > 0 ? mrrResults.reduce((s, r) => s + r.mrr, 0) / mrrResults.length : 0;
+
+  console.log('');
+  const summaryColor = failed === 0 ? '\x1b[32m' : '\x1b[33m';
+  console.log(
+    `${summaryColor}SUMMARY: ${passed}/${results.length} passed | recall=${meanRecall.toFixed(2)} | mrr=${meanMRR.toFixed(2)}\x1b[0m`
+  );
+
+  // Save JSON report
+  const report: EvalReport = {
+    timestamp: new Date().toISOString(),
+    codebasePath: resolvedPath,
+    codegraphSha,
+    summary: { total: results.length, passed, failed, meanRecall, meanMRR },
+    results,
+  };
+
+  const resultsDir = path.join(__dirname, 'results');
+  fs.mkdirSync(resultsDir, { recursive: true });
+  const reportFile = path.join(
+    resultsDir,
+    `${new Date().toISOString().replace(/[:.]/g, '-')}.json`
+  );
+  fs.writeFileSync(reportFile, JSON.stringify(report, null, 2));
+  console.log(`\nReport saved: ${reportFile}`);
+
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});