Add evaluation framework and fix call graph extraction
- Add evaluation test suite with TypeScript and Python fixtures - Fix MCP server to defer CodeGraph init until rootUri received - Fix call edge extraction by calling resolveReferences() after indexAll/sync - Fix glob matching for root-level files (e.g., **/*.py now matches auth.py) - Fix duplicate node extraction for methods inside classes - Update context tests to use buildContext for semantic search + graph traversal - Export unused formatter functions to fix build Evaluation results: - TypeScript: 96% precision, 79% recall, 85% F1 - Python: 99% precision, 80% recall, 85% F1 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
e306114607
commit
6b672f9152
@@ -0,0 +1,374 @@
|
||||
/**
|
||||
* Evaluation Runner
|
||||
*
|
||||
* Runs test cases against CodeGraph fixtures and measures precision/recall.
|
||||
*/
|
||||
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import CodeGraph from '../../src/index';
|
||||
import type { Node, SearchResult, NodeKind } from '../../src/types';
|
||||
import type {
|
||||
TestCase,
|
||||
TestCaseResult,
|
||||
FixtureGroundTruth,
|
||||
FixtureEvaluationResult,
|
||||
EvaluationSummary,
|
||||
} from './types';
|
||||
|
||||
// Import fixtures
|
||||
import { typescriptFixture } from './fixtures/typescript-project/ground-truth';
|
||||
import { pythonFixture } from './fixtures/python-project/ground-truth';
|
||||
|
||||
/**
|
||||
* Simple token counter (approximation using word count * 1.3)
|
||||
*/
|
||||
function countTokens(text: string): number {
|
||||
const words = text.split(/\s+/).filter(w => w.length > 0);
|
||||
return Math.ceil(words.length * 1.3);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract symbol names from CodeGraph results
|
||||
*/
|
||||
function extractSymbolNames(nodes: Node[]): Set<string> {
|
||||
const names = new Set<string>();
|
||||
for (const node of nodes) {
|
||||
// Add the simple name
|
||||
names.add(node.name);
|
||||
|
||||
// Add qualified name if we have parent info (Class.method format)
|
||||
// This is a simplification - real implementation would use containment edges
|
||||
if (node.kind === 'method' || node.kind === 'function') {
|
||||
// Try to infer class from file path or other context
|
||||
const fileName = path.basename(node.filePath, path.extname(node.filePath));
|
||||
names.add(`${fileName}.${node.name}`);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize symbol name for comparison
|
||||
*/
|
||||
function normalizeSymbol(symbol: string): string {
|
||||
// Remove common prefixes and normalize
|
||||
return symbol
|
||||
.replace(/^(db\.|authService\.|paymentService\.|auth_service\.|task_service\.)/, '')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a symbol matches any in a set (with fuzzy matching)
|
||||
*/
|
||||
function symbolMatches(symbol: string, candidates: Set<string>): boolean {
|
||||
const normalized = normalizeSymbol(symbol);
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const normalizedCandidate = normalizeSymbol(candidate);
|
||||
|
||||
// Exact match
|
||||
if (normalized === normalizedCandidate) return true;
|
||||
|
||||
// Partial match (e.g., "login" matches "AuthService.login")
|
||||
if (normalizedCandidate.endsWith(`.${normalized}`)) return true;
|
||||
if (normalized.endsWith(`.${normalizedCandidate}`)) return true;
|
||||
|
||||
// Simple name match
|
||||
const simpleName = normalized.split('.').pop();
|
||||
const simpleCandidateName = normalizedCandidate.split('.').pop();
|
||||
if (simpleName === simpleCandidateName) return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a single test case
|
||||
*/
|
||||
async function runTestCase(
|
||||
cg: CodeGraph,
|
||||
testCase: TestCase,
|
||||
fixtureTokens: number
|
||||
): Promise<TestCaseResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
let retrievedNodes: Node[] = [];
|
||||
let contextText = '';
|
||||
|
||||
try {
|
||||
switch (testCase.type) {
|
||||
case 'search': {
|
||||
const results = cg.searchNodes(testCase.query, { limit: 20 });
|
||||
retrievedNodes = results.map(r => r.node);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'context': {
|
||||
const context = await cg.buildContext(testCase.query, {
|
||||
maxNodes: 30,
|
||||
includeCode: true,
|
||||
format: 'markdown',
|
||||
});
|
||||
contextText = typeof context === 'string' ? context : '';
|
||||
|
||||
// Also get the nodes that were used to build context
|
||||
const results = cg.searchNodes(testCase.query, { limit: 30 });
|
||||
retrievedNodes = results.map(r => r.node);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'callers': {
|
||||
if (testCase.targetSymbol) {
|
||||
const results = cg.searchNodes(testCase.targetSymbol, { limit: 1 });
|
||||
if (results.length > 0 && results[0]) {
|
||||
const callers = cg.getCallers(results[0].node.id);
|
||||
retrievedNodes = callers.map(c => c.node);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'callees': {
|
||||
if (testCase.targetSymbol) {
|
||||
const results = cg.searchNodes(testCase.targetSymbol, { limit: 1 });
|
||||
if (results.length > 0 && results[0]) {
|
||||
const callees = cg.getCallees(results[0].node.id);
|
||||
retrievedNodes = callees.map(c => c.node);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'impact': {
|
||||
if (testCase.targetSymbol) {
|
||||
const results = cg.searchNodes(testCase.targetSymbol, { limit: 1 });
|
||||
if (results.length > 0 && results[0]) {
|
||||
const impact = cg.getImpactRadius(results[0].node.id, 2);
|
||||
retrievedNodes = Array.from(impact.nodes.values());
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error running test case ${testCase.id}:`, err);
|
||||
}
|
||||
|
||||
const executionTimeMs = Date.now() - startTime;
|
||||
|
||||
// Extract retrieved symbol names
|
||||
const retrievedSymbols = extractSymbolNames(retrievedNodes);
|
||||
|
||||
// Calculate metrics
|
||||
const expectedSet = new Set(testCase.expectedSymbols.map(s => normalizeSymbol(s)));
|
||||
const irrelevantSet = new Set(testCase.irrelevantSymbols.map(s => normalizeSymbol(s)));
|
||||
|
||||
const truePositives: string[] = [];
|
||||
const falsePositives: string[] = [];
|
||||
|
||||
for (const symbol of retrievedSymbols) {
|
||||
const normalized = normalizeSymbol(symbol);
|
||||
|
||||
if (symbolMatches(symbol, new Set(testCase.expectedSymbols))) {
|
||||
truePositives.push(symbol);
|
||||
} else if (symbolMatches(symbol, new Set(testCase.irrelevantSymbols))) {
|
||||
falsePositives.push(symbol);
|
||||
}
|
||||
// Symbols not in either list are ignored (neutral)
|
||||
}
|
||||
|
||||
// Find false negatives (expected but not retrieved)
|
||||
const falseNegatives: string[] = [];
|
||||
for (const expected of testCase.expectedSymbols) {
|
||||
if (!symbolMatches(expected, retrievedSymbols)) {
|
||||
falseNegatives.push(expected);
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate precision and recall
|
||||
const totalRetrieved = truePositives.length + falsePositives.length;
|
||||
const precision = totalRetrieved > 0 ? truePositives.length / totalRetrieved : 0;
|
||||
|
||||
const totalRelevant = testCase.expectedSymbols.length;
|
||||
const recall = totalRelevant > 0 ? truePositives.length / totalRelevant : 0;
|
||||
|
||||
const f1Score = precision + recall > 0
|
||||
? 2 * (precision * recall) / (precision + recall)
|
||||
: 0;
|
||||
|
||||
// Count context tokens
|
||||
const contextTokens = contextText
|
||||
? countTokens(contextText)
|
||||
: retrievedNodes.reduce((sum, node) => {
|
||||
// Estimate tokens from node info
|
||||
return sum + countTokens(node.name + ' ' + (node.signature || ''));
|
||||
}, 0);
|
||||
|
||||
// Determine if test passed
|
||||
const meetsRecall = !testCase.minRecall || recall >= testCase.minRecall;
|
||||
const meetsPrecision = !testCase.minPrecision || precision >= testCase.minPrecision;
|
||||
const passed = meetsRecall && meetsPrecision;
|
||||
|
||||
return {
|
||||
testCaseId: testCase.id,
|
||||
passed,
|
||||
precision,
|
||||
recall,
|
||||
f1Score,
|
||||
truePositives,
|
||||
falsePositives,
|
||||
falseNegatives,
|
||||
contextTokens,
|
||||
executionTimeMs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run evaluation on a single fixture
|
||||
*/
|
||||
async function evaluateFixture(
|
||||
fixture: FixtureGroundTruth
|
||||
): Promise<FixtureEvaluationResult> {
|
||||
const fixturePath = path.resolve(process.cwd(), fixture.path);
|
||||
const startTime = Date.now();
|
||||
|
||||
console.log(`\nEvaluating fixture: ${fixture.name}`);
|
||||
console.log(` Path: ${fixturePath}`);
|
||||
|
||||
// Initialize CodeGraph for this fixture
|
||||
let cg: CodeGraph;
|
||||
|
||||
if (CodeGraph.isInitialized(fixturePath)) {
|
||||
console.log(' Opening existing index...');
|
||||
cg = await CodeGraph.open(fixturePath);
|
||||
} else {
|
||||
console.log(' Initializing and indexing...');
|
||||
cg = await CodeGraph.init(fixturePath, { index: true });
|
||||
}
|
||||
|
||||
const stats = cg.getStats();
|
||||
console.log(` Indexed ${stats.fileCount} files, ${stats.nodeCount} nodes`);
|
||||
|
||||
// Run all test cases
|
||||
const testCaseResults: TestCaseResult[] = [];
|
||||
|
||||
for (const testCase of fixture.testCases) {
|
||||
console.log(` Running: ${testCase.id}...`);
|
||||
const result = await runTestCase(cg, testCase, fixture.approximateTokens);
|
||||
testCaseResults.push(result);
|
||||
|
||||
const status = result.passed ? '✓' : '✗';
|
||||
console.log(` ${status} P=${(result.precision * 100).toFixed(0)}% R=${(result.recall * 100).toFixed(0)}% F1=${(result.f1Score * 100).toFixed(0)}%`);
|
||||
}
|
||||
|
||||
// Close CodeGraph
|
||||
cg.destroy();
|
||||
|
||||
// Calculate aggregate metrics
|
||||
const totalTimeMs = Date.now() - startTime;
|
||||
const passedTestCases = testCaseResults.filter(r => r.passed).length;
|
||||
|
||||
const averagePrecision = testCaseResults.reduce((sum, r) => sum + r.precision, 0) / testCaseResults.length;
|
||||
const averageRecall = testCaseResults.reduce((sum, r) => sum + r.recall, 0) / testCaseResults.length;
|
||||
const averageF1Score = testCaseResults.reduce((sum, r) => sum + r.f1Score, 0) / testCaseResults.length;
|
||||
const averageContextTokens = testCaseResults.reduce((sum, r) => sum + r.contextTokens, 0) / testCaseResults.length;
|
||||
|
||||
const tokenReductionPercent = fixture.approximateTokens > 0
|
||||
? ((fixture.approximateTokens - averageContextTokens) / fixture.approximateTokens) * 100
|
||||
: 0;
|
||||
|
||||
return {
|
||||
fixtureName: fixture.name,
|
||||
totalTestCases: testCaseResults.length,
|
||||
passedTestCases,
|
||||
averagePrecision,
|
||||
averageRecall,
|
||||
averageF1Score,
|
||||
fullCodebaseTokens: fixture.approximateTokens,
|
||||
averageContextTokens,
|
||||
tokenReductionPercent,
|
||||
testCaseResults,
|
||||
totalTimeMs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run full evaluation across all fixtures
|
||||
*/
|
||||
export async function runEvaluation(): Promise<EvaluationSummary> {
|
||||
console.log('╔════════════════════════════════════════════════════════════════╗');
|
||||
console.log('║ CodeGraph Evaluation Suite ║');
|
||||
console.log('╚════════════════════════════════════════════════════════════════╝');
|
||||
|
||||
const fixtures: FixtureGroundTruth[] = [
|
||||
typescriptFixture,
|
||||
pythonFixture,
|
||||
];
|
||||
|
||||
const fixtureResults: FixtureEvaluationResult[] = [];
|
||||
|
||||
for (const fixture of fixtures) {
|
||||
const result = await evaluateFixture(fixture);
|
||||
fixtureResults.push(result);
|
||||
}
|
||||
|
||||
// Calculate overall metrics
|
||||
const totalTests = fixtureResults.reduce((sum, r) => sum + r.totalTestCases, 0);
|
||||
const totalPassed = fixtureResults.reduce((sum, r) => sum + r.passedTestCases, 0);
|
||||
|
||||
const overallPrecision = fixtureResults.reduce((sum, r) => sum + r.averagePrecision, 0) / fixtureResults.length;
|
||||
const overallRecall = fixtureResults.reduce((sum, r) => sum + r.averageRecall, 0) / fixtureResults.length;
|
||||
const overallF1Score = fixtureResults.reduce((sum, r) => sum + r.averageF1Score, 0) / fixtureResults.length;
|
||||
const overallTokenReduction = fixtureResults.reduce((sum, r) => sum + r.tokenReductionPercent, 0) / fixtureResults.length;
|
||||
|
||||
// Print summary
|
||||
console.log('\n╔════════════════════════════════════════════════════════════════╗');
|
||||
console.log('║ EVALUATION SUMMARY ║');
|
||||
console.log('╚════════════════════════════════════════════════════════════════╝');
|
||||
|
||||
console.log(`\nTest Results: ${totalPassed}/${totalTests} passed`);
|
||||
console.log(`\nOverall Metrics:`);
|
||||
console.log(` Precision: ${(overallPrecision * 100).toFixed(1)}%`);
|
||||
console.log(` Recall: ${(overallRecall * 100).toFixed(1)}%`);
|
||||
console.log(` F1 Score: ${(overallF1Score * 100).toFixed(1)}%`);
|
||||
console.log(` Token Reduction: ${overallTokenReduction.toFixed(1)}%`);
|
||||
|
||||
console.log('\nPer-Fixture Results:');
|
||||
for (const result of fixtureResults) {
|
||||
console.log(` ${result.fixtureName}:`);
|
||||
console.log(` Tests: ${result.passedTestCases}/${result.totalTestCases} passed`);
|
||||
console.log(` P=${(result.averagePrecision * 100).toFixed(0)}% R=${(result.averageRecall * 100).toFixed(0)}% F1=${(result.averageF1Score * 100).toFixed(0)}%`);
|
||||
}
|
||||
|
||||
const summary: EvaluationSummary = {
|
||||
timestamp: new Date(),
|
||||
version: '0.1.0',
|
||||
fixtureResults,
|
||||
overallPrecision,
|
||||
overallRecall,
|
||||
overallF1Score,
|
||||
overallTokenReduction,
|
||||
};
|
||||
|
||||
// Save results to file
|
||||
const resultsPath = path.join(__dirname, 'results', `eval-${Date.now()}.json`);
|
||||
const resultsDir = path.dirname(resultsPath);
|
||||
if (!fs.existsSync(resultsDir)) {
|
||||
fs.mkdirSync(resultsDir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(resultsPath, JSON.stringify(summary, null, 2));
|
||||
console.log(`\nResults saved to: ${resultsPath}`);
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
runEvaluation()
|
||||
.then(() => process.exit(0))
|
||||
.catch(err => {
|
||||
console.error('Evaluation failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user