Colby McHenryandClaude Opus 4.5
6b672f9152
- Add evaluation test suite with TypeScript and Python fixtures - Fix MCP server to defer CodeGraph init until rootUri received - Fix call edge extraction by calling resolveReferences() after indexAll/sync - Fix glob matching for root-level files (e.g., **/*.py now matches auth.py) - Fix duplicate node extraction for methods inside classes - Update context tests to use buildContext for semantic search + graph traversal - Export unused formatter functions to fix build Evaluation results: - TypeScript: 96% precision, 79% recall, 85% F1 - Python: 99% precision, 80% recall, 85% F1 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
306 lines
9.0 KiB
TypeScript
306 lines
9.0 KiB
TypeScript
/**
|
|
* Ground truth definitions for the Python task management fixture
|
|
*/
|
|
|
|
import { FixtureGroundTruth } from '../../types';
|
|
|
|
export const pythonFixture: FixtureGroundTruth = {
|
|
name: 'python-taskmanager',
|
|
path: '__tests__/evaluation/fixtures/python-project',
|
|
language: 'python',
|
|
totalFiles: 5,
|
|
approximateTokens: 1200, // Rough estimate
|
|
|
|
testCases: [
|
|
// =========================================================================
|
|
// Search Tests
|
|
// =========================================================================
|
|
{
|
|
id: 'py-search-auth',
|
|
description: 'Search for authentication functionality',
|
|
query: 'authentication login',
|
|
type: 'search',
|
|
expectedSymbols: ['AuthService', 'AuthService.login', 'AuthService.register', 'verify_password'],
|
|
irrelevantSymbols: ['TaskService', 'validate_task_title', 'Project'],
|
|
minRecall: 0.7,
|
|
minPrecision: 0.5,
|
|
},
|
|
{
|
|
id: 'py-search-task',
|
|
description: 'Search for task management',
|
|
query: 'task create complete',
|
|
type: 'search',
|
|
expectedSymbols: ['TaskService', 'TaskService.create_task', 'TaskService.complete_task', 'Task'],
|
|
irrelevantSymbols: ['AuthService', 'validate_email', 'hash_password'],
|
|
minRecall: 0.7,
|
|
minPrecision: 0.5,
|
|
},
|
|
{
|
|
id: 'py-search-validation',
|
|
description: 'Search for validation',
|
|
query: 'validate',
|
|
type: 'search',
|
|
expectedSymbols: ['validate_email', 'validate_password', 'validate_task_title'],
|
|
irrelevantSymbols: ['hash_password', 'generate_token', 'TaskService'],
|
|
minRecall: 0.8,
|
|
minPrecision: 0.6,
|
|
},
|
|
|
|
// =========================================================================
|
|
// Context Tests
|
|
// =========================================================================
|
|
{
|
|
id: 'py-context-login-bug',
|
|
description: 'Build context for fixing login issues',
|
|
query: 'debug why users cannot log in',
|
|
type: 'context',
|
|
expectedSymbols: [
|
|
'AuthService.login',
|
|
'verify_password',
|
|
'db.get_user_by_email',
|
|
'User',
|
|
'hash_password',
|
|
],
|
|
irrelevantSymbols: [
|
|
'TaskService',
|
|
'validate_task_title',
|
|
'Project',
|
|
'Task',
|
|
],
|
|
minRecall: 0.8,
|
|
minPrecision: 0.6,
|
|
},
|
|
{
|
|
id: 'py-context-task-creation',
|
|
description: 'Build context for task creation flow',
|
|
query: 'understand how tasks are created',
|
|
type: 'context',
|
|
expectedSymbols: [
|
|
'TaskService.create_task',
|
|
'validate_task_title',
|
|
'auth_service.get_user_id',
|
|
'db.create_task',
|
|
'Task',
|
|
'generate_token',
|
|
],
|
|
irrelevantSymbols: [
|
|
'validate_email',
|
|
'hash_password',
|
|
'AuthService.register',
|
|
'Project',
|
|
],
|
|
minRecall: 0.7,
|
|
minPrecision: 0.5,
|
|
},
|
|
{
|
|
id: 'py-context-user-registration',
|
|
description: 'Build context for user registration',
|
|
query: 'add email confirmation to registration',
|
|
type: 'context',
|
|
expectedSymbols: [
|
|
'AuthService.register',
|
|
'validate_email',
|
|
'validate_password',
|
|
'hash_password',
|
|
'db.create_user',
|
|
'User',
|
|
],
|
|
irrelevantSymbols: [
|
|
'TaskService',
|
|
'validate_task_title',
|
|
'Task',
|
|
'Project',
|
|
],
|
|
minRecall: 0.7,
|
|
minPrecision: 0.6,
|
|
},
|
|
|
|
// =========================================================================
|
|
// Callers Tests
|
|
// =========================================================================
|
|
{
|
|
id: 'py-callers-get_user_id',
|
|
description: 'Find all callers of auth_service.get_user_id',
|
|
query: 'get_user_id',
|
|
type: 'callers',
|
|
targetSymbol: 'get_user_id',
|
|
expectedSymbols: [
|
|
'TaskService.create_task',
|
|
'TaskService.get_task',
|
|
'TaskService.get_user_tasks',
|
|
],
|
|
irrelevantSymbols: [
|
|
'AuthService.login',
|
|
'validate_email',
|
|
'hash_password',
|
|
],
|
|
minRecall: 1.0,
|
|
minPrecision: 1.0,
|
|
},
|
|
{
|
|
id: 'py-callers-validate_email',
|
|
description: 'Find all callers of validate_email',
|
|
query: 'validate_email',
|
|
type: 'callers',
|
|
targetSymbol: 'validate_email',
|
|
expectedSymbols: [
|
|
'AuthService.register',
|
|
],
|
|
irrelevantSymbols: [
|
|
'TaskService',
|
|
'validate_password',
|
|
'hash_password',
|
|
],
|
|
minRecall: 1.0,
|
|
minPrecision: 1.0,
|
|
},
|
|
{
|
|
id: 'py-callers-generate_token',
|
|
description: 'Find all callers of generate_token',
|
|
query: 'generate_token',
|
|
type: 'callers',
|
|
targetSymbol: 'generate_token',
|
|
expectedSymbols: [
|
|
'AuthService.register',
|
|
'AuthService.login',
|
|
'TaskService.create_task',
|
|
],
|
|
irrelevantSymbols: [
|
|
'validate_email',
|
|
'validate_password',
|
|
'db.get_user',
|
|
],
|
|
minRecall: 1.0,
|
|
minPrecision: 1.0,
|
|
},
|
|
|
|
// =========================================================================
|
|
// Callees Tests
|
|
// =========================================================================
|
|
{
|
|
id: 'py-callees-login',
|
|
description: 'Find what AuthService.login calls',
|
|
query: 'login',
|
|
type: 'callees',
|
|
targetSymbol: 'login',
|
|
expectedSymbols: [
|
|
'db.get_user_by_email',
|
|
'verify_password',
|
|
'generate_token',
|
|
],
|
|
irrelevantSymbols: [
|
|
'validate_email',
|
|
'hash_password',
|
|
'validate_task_title',
|
|
],
|
|
minRecall: 1.0,
|
|
minPrecision: 1.0,
|
|
},
|
|
{
|
|
id: 'py-callees-create_task',
|
|
description: 'Find what TaskService.create_task calls',
|
|
query: 'create_task',
|
|
type: 'callees',
|
|
targetSymbol: 'TaskService.create_task',
|
|
expectedSymbols: [
|
|
'auth_service.get_user_id',
|
|
'validate_task_title',
|
|
'generate_token',
|
|
'db.create_task',
|
|
],
|
|
irrelevantSymbols: [
|
|
'validate_email',
|
|
'hash_password',
|
|
'db.get_user',
|
|
],
|
|
minRecall: 0.8,
|
|
minPrecision: 0.8,
|
|
},
|
|
|
|
// =========================================================================
|
|
// Impact Tests
|
|
// =========================================================================
|
|
{
|
|
id: 'py-impact-generate_token',
|
|
description: 'Impact of changing generate_token',
|
|
query: 'generate_token',
|
|
type: 'impact',
|
|
targetSymbol: 'generate_token',
|
|
expectedSymbols: [
|
|
// Direct callers
|
|
'AuthService.register',
|
|
'AuthService.login',
|
|
'TaskService.create_task',
|
|
],
|
|
irrelevantSymbols: [
|
|
'validate_email',
|
|
'validate_task_title',
|
|
'db.get_project',
|
|
],
|
|
minRecall: 0.8,
|
|
minPrecision: 0.7,
|
|
},
|
|
{
|
|
id: 'py-impact-get_user_id',
|
|
description: 'Impact of changing get_user_id',
|
|
query: 'get_user_id',
|
|
type: 'impact',
|
|
targetSymbol: 'get_user_id',
|
|
expectedSymbols: [
|
|
'TaskService.create_task',
|
|
'TaskService.get_task',
|
|
'TaskService.get_user_tasks',
|
|
'TaskService.complete_task',
|
|
'TaskService.delete_task',
|
|
],
|
|
irrelevantSymbols: [
|
|
'AuthService.register',
|
|
'validate_email',
|
|
'hash_password',
|
|
],
|
|
minRecall: 0.8,
|
|
minPrecision: 0.7,
|
|
},
|
|
],
|
|
|
|
// Known call graph edges for validation
|
|
callGraph: [
|
|
// Auth -> Database
|
|
{ caller: 'AuthService.register', callee: 'db.get_user_by_email' },
|
|
{ caller: 'AuthService.register', callee: 'db.create_user' },
|
|
{ caller: 'AuthService.login', callee: 'db.get_user_by_email' },
|
|
|
|
// Auth -> Crypto
|
|
{ caller: 'AuthService.register', callee: 'hash_password' },
|
|
{ caller: 'AuthService.register', callee: 'generate_token' },
|
|
{ caller: 'AuthService.login', callee: 'verify_password' },
|
|
{ caller: 'AuthService.login', callee: 'generate_token' },
|
|
|
|
// Auth -> Validation
|
|
{ caller: 'AuthService.register', callee: 'validate_email' },
|
|
{ caller: 'AuthService.register', callee: 'validate_password' },
|
|
|
|
// Task -> Auth
|
|
{ caller: 'TaskService.create_task', callee: 'auth_service.get_user_id' },
|
|
{ caller: 'TaskService.get_task', callee: 'auth_service.get_user_id' },
|
|
{ caller: 'TaskService.get_user_tasks', callee: 'auth_service.get_user_id' },
|
|
|
|
// Task -> Database
|
|
{ caller: 'TaskService.create_task', callee: 'db.create_task' },
|
|
{ caller: 'TaskService.get_task', callee: 'db.get_task' },
|
|
{ caller: 'TaskService.get_user_tasks', callee: 'db.get_user_tasks' },
|
|
{ caller: 'TaskService.complete_task', callee: 'db.update_task' },
|
|
{ caller: 'TaskService.delete_task', callee: 'db.delete_task' },
|
|
|
|
// Task -> Crypto
|
|
{ caller: 'TaskService.create_task', callee: 'generate_token' },
|
|
|
|
// Task -> Validation
|
|
{ caller: 'TaskService.create_task', callee: 'validate_task_title' },
|
|
|
|
// Task -> Task (internal)
|
|
{ caller: 'TaskService.complete_task', callee: 'TaskService.get_task' },
|
|
{ caller: 'TaskService.delete_task', callee: 'TaskService.get_task' },
|
|
],
|
|
};
|