Add evaluation framework and fix call graph extraction

- Add evaluation test suite with TypeScript and Python fixtures
- Fix MCP server to defer CodeGraph init until rootUri received
- Fix call edge extraction by calling resolveReferences() after indexAll/sync
- Fix glob matching for root-level files (e.g., **/*.py now matches auth.py)
- Fix duplicate node extraction for methods inside classes
- Update context tests to use buildContext for semantic search + graph traversal
- Export unused formatter functions to fix build

Evaluation results:
- TypeScript: 96% precision, 79% recall, 85% F1
- Python: 99% precision, 80% recall, 85% F1

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
2026-01-18 18:48:22 -06:00
co-authored by Claude Opus 4.5
parent e306114607
commit 6b672f9152
30 changed files with 2600 additions and 129 deletions
+42 -36
View File
@@ -9,58 +9,64 @@ import { Node, Edge, TaskContext, Subgraph } from '../types';
/**
* Format context as markdown
*
* Creates a structured markdown document optimized for Claude:
* - Summary section
* - Structure tree showing relationships
* - Code blocks with syntax highlighting
* - Related files list
* Creates a compact markdown document optimized for Claude with minimal context usage:
* - Brief summary
* - Entry points with locations
* - Code blocks only for key symbols
*/
export function formatContextAsMarkdown(context: TaskContext): string {
const lines: string[] = [];
// Header
// Header with query
lines.push('## Code Context\n');
// Summary
lines.push(`**Query:** ${context.query}\n`);
lines.push(context.summary + '\n');
// Structure section
lines.push('### Structure\n');
lines.push('```');
lines.push(formatSubgraphTree(context.subgraph, context.entryPoints));
lines.push('```\n');
// Entry points - compact format
if (context.entryPoints.length > 0) {
lines.push('### Entry Points\n');
for (const node of context.entryPoints) {
const location = node.startLine ? `:${node.startLine}` : '';
lines.push(`- **${node.name}** (${node.kind}) - ${node.filePath}${location}`);
if (node.signature) {
lines.push(` \`${node.signature}\``);
}
}
lines.push('');
}
// Code blocks section
// Related symbols - compact list (skip verbose structure tree)
const otherSymbols = Array.from(context.subgraph.nodes.values())
.filter(n => !context.entryPoints.some(e => e.id === n.id))
.slice(0, 10); // Limit to 10 related symbols
if (otherSymbols.length > 0) {
lines.push('### Related Symbols\n');
const byFile = new Map<string, Node[]>();
for (const node of otherSymbols) {
const existing = byFile.get(node.filePath) || [];
existing.push(node);
byFile.set(node.filePath, existing);
}
for (const [file, nodes] of byFile) {
const nodeList = nodes.map(n => `${n.name}:${n.startLine}`).join(', ');
lines.push(`- ${file}: ${nodeList}`);
}
lines.push('');
}
// Code blocks - only for key entry points
if (context.codeBlocks.length > 0) {
lines.push('### Code\n');
for (const block of context.codeBlocks) {
const nodeName = block.node?.name ?? 'Unknown';
const nodeKind = block.node?.kind ?? 'unknown';
lines.push(`#### ${nodeName} (${nodeKind}) - ${block.filePath}:${block.startLine}\n`);
lines.push(`#### ${nodeName} (${block.filePath}:${block.startLine})\n`);
lines.push('```' + block.language);
lines.push(block.content);
lines.push('```\n');
}
}
// Related files section
if (context.relatedFiles.length > 0) {
lines.push('### Related Files\n');
for (const file of context.relatedFiles) {
lines.push(`- ${file}`);
}
lines.push('');
}
// Stats footer
lines.push('---');
lines.push(
`*Context: ${context.stats.nodeCount} symbols, ${context.stats.edgeCount} relationships, ` +
`${context.stats.fileCount} files, ${context.stats.codeBlockCount} code blocks ` +
`(${formatBytes(context.stats.totalCodeSize)})*`
);
return lines.join('\n');
}
@@ -96,7 +102,7 @@ export function formatContextAsJson(context: TaskContext): string {
/**
* Format a subgraph as an ASCII tree structure
*/
function formatSubgraphTree(subgraph: Subgraph, entryPoints: Node[]): string {
export function formatSubgraphTree(subgraph: Subgraph, entryPoints: Node[]): string {
const lines: string[] = [];
const printed = new Set<string>();
@@ -254,7 +260,7 @@ function truncate(str: string, maxLength: number): string {
/**
* Format bytes as human-readable string
*/
function formatBytes(bytes: number): string {
export function formatBytes(bytes: number): string {
if (bytes < 1024) {
return `${bytes} bytes`;
} else if (bytes < 1024 * 1024) {
+13 -8
View File
@@ -26,15 +26,20 @@ import { logDebug, logWarn } from '../errors';
/**
* Default options for context building
*
* Tuned for minimal context usage while still providing useful results:
* - Fewer nodes and code blocks by default
* - Smaller code block size limit
* - Shallower traversal
*/
const DEFAULT_BUILD_OPTIONS: Required<BuildContextOptions> = {
maxNodes: 50,
maxCodeBlocks: 10,
maxCodeBlockSize: 2000,
maxNodes: 20, // Reduced from 50 - most tasks don't need 50 symbols
maxCodeBlocks: 5, // Reduced from 10 - only show most relevant code
maxCodeBlockSize: 1500, // Reduced from 2000
includeCode: true,
format: 'markdown',
searchLimit: 5,
traversalDepth: 2,
searchLimit: 3, // Reduced from 5 - fewer entry points
traversalDepth: 1, // Reduced from 2 - shallower graph expansion
minScore: 0.3,
};
@@ -42,9 +47,9 @@ const DEFAULT_BUILD_OPTIONS: Required<BuildContextOptions> = {
* Default options for finding relevant context
*/
const DEFAULT_FIND_OPTIONS: Required<FindRelevantContextOptions> = {
searchLimit: 5,
traversalDepth: 2,
maxNodes: 50,
searchLimit: 3, // Reduced from 5
traversalDepth: 1, // Reduced from 2
maxNodes: 20, // Reduced from 50
minScore: 0.3,
edgeKinds: [],
nodeKinds: [],