feat: Add comprehensive evaluation framework for CodeGraph API testing

Introduces automated testing infrastructure to measure CodeGraph performance across searchNodes and findRelevantContext APIs. Includes recall/MRR scoring metrics, predefined test cases for symbol lookup and context exploration, and JSON report generation. Enhances context building with acronym extraction, definition prefix matching, and improved FTS filtering to exclude imports by default.
This commit is contained in:
Colby McHenry
2026-04-06 13:24:27 -05:00
parent d4258b1651
commit 13d3ff3613
5 changed files with 403 additions and 1 deletions
+82
View File
@@ -0,0 +1,82 @@
import type { EvalResult } from './types.js';
export const PASS_THRESHOLD = 0.5;
export function scoreSearchNodes(
caseId: string,
expectedSymbols: string[],
results: Array<{ node: { name: string }; score: number }>,
latencyMs: number
): EvalResult {
const expectedLower = expectedSymbols.map((s) => s.toLowerCase());
const resultNames = results.map((r) => r.node.name.toLowerCase());
const found: string[] = [];
const missed: string[] = [];
let firstRank = 0;
for (let i = 0; i < expectedLower.length; i++) {
const idx = resultNames.indexOf(expectedLower[i]);
if (idx !== -1) {
found.push(expectedSymbols[i]);
if (firstRank === 0) firstRank = idx + 1;
} else {
missed.push(expectedSymbols[i]);
}
}
const recall = expectedSymbols.length > 0 ? found.length / expectedSymbols.length : 0;
const mrr = firstRank > 0 ? 1 / firstRank : 0;
return {
caseId,
pass: recall >= PASS_THRESHOLD,
recall,
mrr,
foundSymbols: found,
missedSymbols: missed,
latencyMs,
};
}
export function scoreFindRelevantContext(
caseId: string,
expectedSymbols: string[],
subgraph: { nodes: Map<string, { name: string }>; edges: unknown[]; roots: string[] },
latencyMs: number
): EvalResult {
const expectedLower = new Set(expectedSymbols.map((s) => s.toLowerCase()));
const nodeNames = new Set<string>();
for (const node of subgraph.nodes.values()) {
nodeNames.add(node.name.toLowerCase());
}
const found: string[] = [];
const missed: string[] = [];
for (const sym of expectedSymbols) {
if (nodeNames.has(sym.toLowerCase())) {
found.push(sym);
} else {
missed.push(sym);
}
}
const recall = expectedSymbols.length > 0 ? found.length / expectedSymbols.length : 0;
const nodeCount = subgraph.nodes.size;
const edgeCount = subgraph.edges.length;
const edgeDensity = nodeCount > 0 ? edgeCount / nodeCount : 0;
return {
caseId,
pass: recall >= PASS_THRESHOLD,
recall,
mrr: 0,
foundSymbols: found,
missedSymbols: missed,
nodeCount,
edgeCount,
edgeDensity,
latencyMs,
};
}