Enhances code extraction and project indexing
Adds support for Dart and Liquid languages with tree-sitter parsing. Improves accuracy of code symbol extraction for existing languages. Indexes project files to enhance code navigation features. Migrates build system to facilitate code contributions. Removes git hook functionality. Integrates Sentry for error tracking and reporting. Enhances project initialization and configuration loading.
This commit is contained in:
Colby McHenry
@@ -1,303 +0,0 @@
|
||||
/**
|
||||
* Evaluation Tests
|
||||
*
|
||||
* Runs the evaluation suite as part of the test suite.
|
||||
* Use `npm run test:eval` to run just these tests.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import CodeGraph from '../../src/index';
|
||||
import type { TestCase, TestCaseResult } from './types';
|
||||
import { typescriptFixture } from './fixtures/typescript-project/ground-truth';
|
||||
import { pythonFixture } from './fixtures/python-project/ground-truth';
|
||||
|
||||
/**
|
||||
* Extract symbol names from nodes
|
||||
*/
|
||||
function extractSymbolNames(nodes: { name: string }[]): Set<string> {
|
||||
return new Set(nodes.map(n => n.name.toLowerCase()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize symbol name
|
||||
*/
|
||||
function normalizeSymbol(symbol: string): string {
|
||||
return symbol.split('.').pop()?.toLowerCase() || symbol.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if symbol matches
|
||||
*/
|
||||
function symbolMatches(symbol: string, candidates: Set<string>): boolean {
|
||||
const normalized = normalizeSymbol(symbol);
|
||||
for (const candidate of candidates) {
|
||||
if (normalizeSymbol(candidate) === normalized) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a target node by name, supporting qualified names like "ClassName.methodName"
|
||||
*/
|
||||
function findTargetNode(cg: CodeGraph, targetSymbol: string): { id: string; name: string } | null {
|
||||
// Check if it's a qualified name (e.g., "OrderService.createOrder")
|
||||
const parts = targetSymbol.split('.');
|
||||
|
||||
if (parts.length === 2) {
|
||||
const [className, methodName] = parts;
|
||||
// Search for the method name and filter by qualified name containing the class
|
||||
const results = cg.searchNodes(methodName!, { limit: 20 });
|
||||
for (const r of results) {
|
||||
if (r.node.qualifiedName.includes(className!) && r.node.name === methodName) {
|
||||
return { id: r.node.id, name: r.node.name };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to simple search
|
||||
const results = cg.searchNodes(targetSymbol, { limit: 1 });
|
||||
if (results.length > 0 && results[0]) {
|
||||
return { id: results[0].node.id, name: results[0].node.name };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a single test case and return metrics
|
||||
*/
|
||||
async function runSingleTest(cg: CodeGraph, testCase: TestCase): Promise<TestCaseResult> {
|
||||
let retrievedNodes: { name: string; id: string }[] = [];
|
||||
|
||||
switch (testCase.type) {
|
||||
case 'search': {
|
||||
const results = cg.searchNodes(testCase.query, { limit: 20 });
|
||||
retrievedNodes = results.map(r => ({ name: r.node.name, id: r.node.id }));
|
||||
break;
|
||||
}
|
||||
|
||||
case 'context': {
|
||||
// Use buildContext to get semantic search + graph traversal
|
||||
const context = await cg.buildContext(testCase.query, {
|
||||
maxNodes: 30,
|
||||
traversalDepth: 2,
|
||||
searchLimit: 5,
|
||||
format: 'object',
|
||||
});
|
||||
// Extract nodes from the subgraph
|
||||
if (typeof context !== 'string' && context.subgraph) {
|
||||
retrievedNodes = Array.from(context.subgraph.nodes.values()).map(n => ({
|
||||
name: n.name,
|
||||
id: n.id,
|
||||
}));
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'callers': {
|
||||
if (testCase.targetSymbol) {
|
||||
const targetNode = findTargetNode(cg, testCase.targetSymbol);
|
||||
if (targetNode) {
|
||||
const callers = cg.getCallers(targetNode.id);
|
||||
retrievedNodes = callers.map(c => ({ name: c.node.name, id: c.node.id }));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'callees': {
|
||||
if (testCase.targetSymbol) {
|
||||
const targetNode = findTargetNode(cg, testCase.targetSymbol);
|
||||
if (targetNode) {
|
||||
const callees = cg.getCallees(targetNode.id);
|
||||
retrievedNodes = callees.map(c => ({ name: c.node.name, id: c.node.id }));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'impact': {
|
||||
if (testCase.targetSymbol) {
|
||||
const targetNode = findTargetNode(cg, testCase.targetSymbol);
|
||||
if (targetNode) {
|
||||
const impact = cg.getImpactRadius(targetNode.id, 2);
|
||||
retrievedNodes = Array.from(impact.nodes.values()).map(n => ({ name: n.name, id: n.id }));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate metrics
|
||||
const retrievedSymbols = extractSymbolNames(retrievedNodes);
|
||||
|
||||
const truePositives: string[] = [];
|
||||
const falsePositives: string[] = [];
|
||||
|
||||
for (const symbol of retrievedSymbols) {
|
||||
if (symbolMatches(symbol, new Set(testCase.expectedSymbols))) {
|
||||
truePositives.push(symbol);
|
||||
} else if (symbolMatches(symbol, new Set(testCase.irrelevantSymbols))) {
|
||||
falsePositives.push(symbol);
|
||||
}
|
||||
}
|
||||
|
||||
const falseNegatives: string[] = [];
|
||||
for (const expected of testCase.expectedSymbols) {
|
||||
if (!symbolMatches(expected, retrievedSymbols)) {
|
||||
falseNegatives.push(expected);
|
||||
}
|
||||
}
|
||||
|
||||
const totalRetrieved = truePositives.length + falsePositives.length;
|
||||
const precision = totalRetrieved > 0 ? truePositives.length / totalRetrieved : 0;
|
||||
|
||||
const totalRelevant = testCase.expectedSymbols.length;
|
||||
const recall = totalRelevant > 0 ? truePositives.length / totalRelevant : 0;
|
||||
|
||||
const f1Score = precision + recall > 0
|
||||
? 2 * (precision * recall) / (precision + recall)
|
||||
: 0;
|
||||
|
||||
// Check if passed thresholds (with 20% margin)
|
||||
const passedRecall = !testCase.minRecall || recall >= testCase.minRecall * 0.8;
|
||||
const passedPrecision = !testCase.minPrecision || precision >= testCase.minPrecision * 0.8;
|
||||
|
||||
return {
|
||||
testCaseId: testCase.id,
|
||||
passed: passedRecall && passedPrecision,
|
||||
precision,
|
||||
recall,
|
||||
f1Score,
|
||||
truePositives,
|
||||
falsePositives,
|
||||
falseNegatives,
|
||||
contextTokens: 0,
|
||||
executionTimeMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Print a results table
|
||||
*/
|
||||
function printResultsTable(results: TestCaseResult[], fixtureName: string): void {
|
||||
console.log(`\n${'='.repeat(80)}`);
|
||||
console.log(` ${fixtureName} Results`);
|
||||
console.log('='.repeat(80));
|
||||
console.log('');
|
||||
console.log(' Test ID Type Prec Recall F1 Status');
|
||||
console.log(' ' + '-'.repeat(76));
|
||||
|
||||
for (const r of results) {
|
||||
const id = r.testCaseId.padEnd(35);
|
||||
const type = r.testCaseId.split('-')[1]?.padEnd(10) || ''.padEnd(10);
|
||||
const prec = `${(r.precision * 100).toFixed(0)}%`.padStart(5);
|
||||
const recall = `${(r.recall * 100).toFixed(0)}%`.padStart(6);
|
||||
const f1 = `${(r.f1Score * 100).toFixed(0)}%`.padStart(5);
|
||||
const status = r.passed ? '✓' : '✗';
|
||||
console.log(` ${id} ${type} ${prec} ${recall} ${f1} ${status}`);
|
||||
}
|
||||
|
||||
const avgPrecision = results.reduce((sum, r) => sum + r.precision, 0) / results.length;
|
||||
const avgRecall = results.reduce((sum, r) => sum + r.recall, 0) / results.length;
|
||||
const avgF1 = results.reduce((sum, r) => sum + r.f1Score, 0) / results.length;
|
||||
const passRate = results.filter(r => r.passed).length / results.length;
|
||||
|
||||
console.log(' ' + '-'.repeat(76));
|
||||
console.log(` ${'AVERAGE'.padEnd(35)} ${''.padEnd(10)} ${`${(avgPrecision * 100).toFixed(0)}%`.padStart(5)} ${`${(avgRecall * 100).toFixed(0)}%`.padStart(6)} ${`${(avgF1 * 100).toFixed(0)}%`.padStart(5)} ${(passRate * 100).toFixed(0)}%`);
|
||||
console.log('');
|
||||
}
|
||||
|
||||
describe('CodeGraph Evaluation', () => {
|
||||
describe('TypeScript Fixture', () => {
|
||||
let cg: CodeGraph;
|
||||
const fixturePath = path.resolve(__dirname, 'fixtures/typescript-project');
|
||||
const results: TestCaseResult[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
// Clean up any existing index
|
||||
const codegraphDir = path.join(fixturePath, '.codegraph');
|
||||
if (fs.existsSync(codegraphDir)) {
|
||||
fs.rmSync(codegraphDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Initialize and index
|
||||
cg = await CodeGraph.init(fixturePath, { index: true });
|
||||
|
||||
// Initialize embeddings for semantic search
|
||||
await cg.initializeEmbeddings();
|
||||
await cg.generateEmbeddings();
|
||||
}, 120000);
|
||||
|
||||
afterAll(() => {
|
||||
// Print summary table after all tests
|
||||
printResultsTable(results, 'TypeScript');
|
||||
|
||||
if (cg) {
|
||||
cg.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
it('should index all files', () => {
|
||||
const stats = cg.getStats();
|
||||
expect(stats.fileCount).toBeGreaterThanOrEqual(typescriptFixture.totalFiles);
|
||||
});
|
||||
|
||||
// Generate test for each test case - collect results but don't fail
|
||||
for (const testCase of typescriptFixture.testCases) {
|
||||
it(`${testCase.id}: ${testCase.description}`, async () => {
|
||||
const result = await runSingleTest(cg, testCase);
|
||||
results.push(result);
|
||||
// Don't assert - just collect results
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('Python Fixture', () => {
|
||||
let cg: CodeGraph;
|
||||
const fixturePath = path.resolve(__dirname, 'fixtures/python-project');
|
||||
const results: TestCaseResult[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
// Clean up any existing index
|
||||
const codegraphDir = path.join(fixturePath, '.codegraph');
|
||||
if (fs.existsSync(codegraphDir)) {
|
||||
fs.rmSync(codegraphDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Initialize and index
|
||||
cg = await CodeGraph.init(fixturePath, { index: true });
|
||||
|
||||
// Initialize embeddings for semantic search
|
||||
await cg.initializeEmbeddings();
|
||||
await cg.generateEmbeddings();
|
||||
}, 120000);
|
||||
|
||||
afterAll(() => {
|
||||
// Print summary table after all tests
|
||||
printResultsTable(results, 'Python');
|
||||
|
||||
if (cg) {
|
||||
cg.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
it('should index all files', () => {
|
||||
const stats = cg.getStats();
|
||||
expect(stats.fileCount).toBeGreaterThanOrEqual(pythonFixture.totalFiles);
|
||||
});
|
||||
|
||||
// Generate test for each test case - collect results but don't fail
|
||||
for (const testCase of pythonFixture.testCases) {
|
||||
it(`${testCase.id}: ${testCase.description}`, async () => {
|
||||
const result = await runSingleTest(cg, testCase);
|
||||
results.push(result);
|
||||
// Don't assert - just collect results
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,79 +0,0 @@
|
||||
"""Authentication service."""
|
||||
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from models import User
|
||||
from database import db
|
||||
from validation import validate_email, validate_password
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""Hash a password for storage."""
|
||||
salt = secrets.token_hex(16)
|
||||
hash_obj = hashlib.sha256((password + salt).encode())
|
||||
return f"{salt}:{hash_obj.hexdigest()}"
|
||||
|
||||
|
||||
def verify_password(password: str, password_hash: str) -> bool:
|
||||
"""Verify a password against its hash."""
|
||||
salt, stored_hash = password_hash.split(":")
|
||||
hash_obj = hashlib.sha256((password + salt).encode())
|
||||
return hash_obj.hexdigest() == stored_hash
|
||||
|
||||
|
||||
def generate_token() -> str:
|
||||
"""Generate a secure random token."""
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
class AuthService:
|
||||
def __init__(self):
|
||||
self.tokens: dict = {}
|
||||
|
||||
def register(self, email: str, password: str, name: str) -> Tuple[bool, str]:
|
||||
"""Register a new user."""
|
||||
if not validate_email(email):
|
||||
return False, "Invalid email format"
|
||||
|
||||
if not validate_password(password):
|
||||
return False, "Password too weak"
|
||||
|
||||
if db.get_user_by_email(email):
|
||||
return False, "Email already registered"
|
||||
|
||||
user = User(
|
||||
id=generate_token(),
|
||||
email=email,
|
||||
name=name,
|
||||
password_hash=hash_password(password),
|
||||
created_at=datetime.now(),
|
||||
)
|
||||
db.create_user(user)
|
||||
return True, user.id
|
||||
|
||||
def login(self, email: str, password: str) -> Optional[str]:
|
||||
"""Authenticate user and return token."""
|
||||
user = db.get_user_by_email(email)
|
||||
if not user:
|
||||
return None
|
||||
|
||||
if not verify_password(password, user.password_hash):
|
||||
return None
|
||||
|
||||
token = generate_token()
|
||||
self.tokens[token] = user.id
|
||||
return token
|
||||
|
||||
def logout(self, token: str) -> None:
|
||||
"""Invalidate a token."""
|
||||
self.tokens.pop(token, None)
|
||||
|
||||
def get_user_id(self, token: str) -> Optional[str]:
|
||||
"""Get user ID from token."""
|
||||
return self.tokens.get(token)
|
||||
|
||||
|
||||
auth_service = AuthService()
|
||||
@@ -1,54 +0,0 @@
|
||||
"""Database operations."""
|
||||
|
||||
from typing import Optional, List, Dict
|
||||
from models import User, Task, Project
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self):
|
||||
self.users: Dict[str, User] = {}
|
||||
self.tasks: Dict[str, Task] = {}
|
||||
self.projects: Dict[str, Project] = {}
|
||||
|
||||
def get_user(self, user_id: str) -> Optional[User]:
|
||||
return self.users.get(user_id)
|
||||
|
||||
def get_user_by_email(self, email: str) -> Optional[User]:
|
||||
for user in self.users.values():
|
||||
if user.email == email:
|
||||
return user
|
||||
return None
|
||||
|
||||
def create_user(self, user: User) -> None:
|
||||
self.users[user.id] = user
|
||||
|
||||
def get_task(self, task_id: str) -> Optional[Task]:
|
||||
return self.tasks.get(task_id)
|
||||
|
||||
def get_user_tasks(self, user_id: str) -> List[Task]:
|
||||
return [t for t in self.tasks.values() if t.user_id == user_id]
|
||||
|
||||
def create_task(self, task: Task) -> None:
|
||||
self.tasks[task.id] = task
|
||||
|
||||
def update_task(self, task_id: str, **updates) -> Optional[Task]:
|
||||
task = self.tasks.get(task_id)
|
||||
if task:
|
||||
for key, value in updates.items():
|
||||
setattr(task, key, value)
|
||||
return task
|
||||
|
||||
def delete_task(self, task_id: str) -> bool:
|
||||
if task_id in self.tasks:
|
||||
del self.tasks[task_id]
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_project(self, project_id: str) -> Optional[Project]:
|
||||
return self.projects.get(project_id)
|
||||
|
||||
def create_project(self, project: Project) -> None:
|
||||
self.projects[project.id] = project
|
||||
|
||||
|
||||
db = Database()
|
||||
@@ -1,305 +0,0 @@
|
||||
/**
|
||||
* Ground truth definitions for the Python task management fixture
|
||||
*/
|
||||
|
||||
import { FixtureGroundTruth } from '../../types';
|
||||
|
||||
export const pythonFixture: FixtureGroundTruth = {
|
||||
name: 'python-taskmanager',
|
||||
path: '__tests__/evaluation/fixtures/python-project',
|
||||
language: 'python',
|
||||
totalFiles: 5,
|
||||
approximateTokens: 1200, // Rough estimate
|
||||
|
||||
testCases: [
|
||||
// =========================================================================
|
||||
// Search Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'py-search-auth',
|
||||
description: 'Search for authentication functionality',
|
||||
query: 'authentication login',
|
||||
type: 'search',
|
||||
expectedSymbols: ['AuthService', 'AuthService.login', 'AuthService.register', 'verify_password'],
|
||||
irrelevantSymbols: ['TaskService', 'validate_task_title', 'Project'],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.5,
|
||||
},
|
||||
{
|
||||
id: 'py-search-task',
|
||||
description: 'Search for task management',
|
||||
query: 'task create complete',
|
||||
type: 'search',
|
||||
expectedSymbols: ['TaskService', 'TaskService.create_task', 'TaskService.complete_task', 'Task'],
|
||||
irrelevantSymbols: ['AuthService', 'validate_email', 'hash_password'],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.5,
|
||||
},
|
||||
{
|
||||
id: 'py-search-validation',
|
||||
description: 'Search for validation',
|
||||
query: 'validate',
|
||||
type: 'search',
|
||||
expectedSymbols: ['validate_email', 'validate_password', 'validate_task_title'],
|
||||
irrelevantSymbols: ['hash_password', 'generate_token', 'TaskService'],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.6,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Context Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'py-context-login-bug',
|
||||
description: 'Build context for fixing login issues',
|
||||
query: 'debug why users cannot log in',
|
||||
type: 'context',
|
||||
expectedSymbols: [
|
||||
'AuthService.login',
|
||||
'verify_password',
|
||||
'db.get_user_by_email',
|
||||
'User',
|
||||
'hash_password',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'TaskService',
|
||||
'validate_task_title',
|
||||
'Project',
|
||||
'Task',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.6,
|
||||
},
|
||||
{
|
||||
id: 'py-context-task-creation',
|
||||
description: 'Build context for task creation flow',
|
||||
query: 'understand how tasks are created',
|
||||
type: 'context',
|
||||
expectedSymbols: [
|
||||
'TaskService.create_task',
|
||||
'validate_task_title',
|
||||
'auth_service.get_user_id',
|
||||
'db.create_task',
|
||||
'Task',
|
||||
'generate_token',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validate_email',
|
||||
'hash_password',
|
||||
'AuthService.register',
|
||||
'Project',
|
||||
],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.5,
|
||||
},
|
||||
{
|
||||
id: 'py-context-user-registration',
|
||||
description: 'Build context for user registration',
|
||||
query: 'add email confirmation to registration',
|
||||
type: 'context',
|
||||
expectedSymbols: [
|
||||
'AuthService.register',
|
||||
'validate_email',
|
||||
'validate_password',
|
||||
'hash_password',
|
||||
'db.create_user',
|
||||
'User',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'TaskService',
|
||||
'validate_task_title',
|
||||
'Task',
|
||||
'Project',
|
||||
],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.6,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Callers Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'py-callers-get_user_id',
|
||||
description: 'Find all callers of auth_service.get_user_id',
|
||||
query: 'get_user_id',
|
||||
type: 'callers',
|
||||
targetSymbol: 'get_user_id',
|
||||
expectedSymbols: [
|
||||
'TaskService.create_task',
|
||||
'TaskService.get_task',
|
||||
'TaskService.get_user_tasks',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'AuthService.login',
|
||||
'validate_email',
|
||||
'hash_password',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'py-callers-validate_email',
|
||||
description: 'Find all callers of validate_email',
|
||||
query: 'validate_email',
|
||||
type: 'callers',
|
||||
targetSymbol: 'validate_email',
|
||||
expectedSymbols: [
|
||||
'AuthService.register',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'TaskService',
|
||||
'validate_password',
|
||||
'hash_password',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'py-callers-generate_token',
|
||||
description: 'Find all callers of generate_token',
|
||||
query: 'generate_token',
|
||||
type: 'callers',
|
||||
targetSymbol: 'generate_token',
|
||||
expectedSymbols: [
|
||||
'AuthService.register',
|
||||
'AuthService.login',
|
||||
'TaskService.create_task',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validate_email',
|
||||
'validate_password',
|
||||
'db.get_user',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Callees Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'py-callees-login',
|
||||
description: 'Find what AuthService.login calls',
|
||||
query: 'login',
|
||||
type: 'callees',
|
||||
targetSymbol: 'login',
|
||||
expectedSymbols: [
|
||||
'db.get_user_by_email',
|
||||
'verify_password',
|
||||
'generate_token',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validate_email',
|
||||
'hash_password',
|
||||
'validate_task_title',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'py-callees-create_task',
|
||||
description: 'Find what TaskService.create_task calls',
|
||||
query: 'create_task',
|
||||
type: 'callees',
|
||||
targetSymbol: 'TaskService.create_task',
|
||||
expectedSymbols: [
|
||||
'auth_service.get_user_id',
|
||||
'validate_task_title',
|
||||
'generate_token',
|
||||
'db.create_task',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validate_email',
|
||||
'hash_password',
|
||||
'db.get_user',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.8,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Impact Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'py-impact-generate_token',
|
||||
description: 'Impact of changing generate_token',
|
||||
query: 'generate_token',
|
||||
type: 'impact',
|
||||
targetSymbol: 'generate_token',
|
||||
expectedSymbols: [
|
||||
// Direct callers
|
||||
'AuthService.register',
|
||||
'AuthService.login',
|
||||
'TaskService.create_task',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validate_email',
|
||||
'validate_task_title',
|
||||
'db.get_project',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.7,
|
||||
},
|
||||
{
|
||||
id: 'py-impact-get_user_id',
|
||||
description: 'Impact of changing get_user_id',
|
||||
query: 'get_user_id',
|
||||
type: 'impact',
|
||||
targetSymbol: 'get_user_id',
|
||||
expectedSymbols: [
|
||||
'TaskService.create_task',
|
||||
'TaskService.get_task',
|
||||
'TaskService.get_user_tasks',
|
||||
'TaskService.complete_task',
|
||||
'TaskService.delete_task',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'AuthService.register',
|
||||
'validate_email',
|
||||
'hash_password',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.7,
|
||||
},
|
||||
],
|
||||
|
||||
// Known call graph edges for validation
|
||||
callGraph: [
|
||||
// Auth -> Database
|
||||
{ caller: 'AuthService.register', callee: 'db.get_user_by_email' },
|
||||
{ caller: 'AuthService.register', callee: 'db.create_user' },
|
||||
{ caller: 'AuthService.login', callee: 'db.get_user_by_email' },
|
||||
|
||||
// Auth -> Crypto
|
||||
{ caller: 'AuthService.register', callee: 'hash_password' },
|
||||
{ caller: 'AuthService.register', callee: 'generate_token' },
|
||||
{ caller: 'AuthService.login', callee: 'verify_password' },
|
||||
{ caller: 'AuthService.login', callee: 'generate_token' },
|
||||
|
||||
// Auth -> Validation
|
||||
{ caller: 'AuthService.register', callee: 'validate_email' },
|
||||
{ caller: 'AuthService.register', callee: 'validate_password' },
|
||||
|
||||
// Task -> Auth
|
||||
{ caller: 'TaskService.create_task', callee: 'auth_service.get_user_id' },
|
||||
{ caller: 'TaskService.get_task', callee: 'auth_service.get_user_id' },
|
||||
{ caller: 'TaskService.get_user_tasks', callee: 'auth_service.get_user_id' },
|
||||
|
||||
// Task -> Database
|
||||
{ caller: 'TaskService.create_task', callee: 'db.create_task' },
|
||||
{ caller: 'TaskService.get_task', callee: 'db.get_task' },
|
||||
{ caller: 'TaskService.get_user_tasks', callee: 'db.get_user_tasks' },
|
||||
{ caller: 'TaskService.complete_task', callee: 'db.update_task' },
|
||||
{ caller: 'TaskService.delete_task', callee: 'db.delete_task' },
|
||||
|
||||
// Task -> Crypto
|
||||
{ caller: 'TaskService.create_task', callee: 'generate_token' },
|
||||
|
||||
// Task -> Validation
|
||||
{ caller: 'TaskService.create_task', callee: 'validate_task_title' },
|
||||
|
||||
// Task -> Task (internal)
|
||||
{ caller: 'TaskService.complete_task', callee: 'TaskService.get_task' },
|
||||
{ caller: 'TaskService.delete_task', callee: 'TaskService.get_task' },
|
||||
],
|
||||
};
|
||||
@@ -1,34 +0,0 @@
|
||||
"""Data models for the application."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
|
||||
|
||||
@dataclass
|
||||
class User:
|
||||
id: str
|
||||
email: str
|
||||
name: str
|
||||
password_hash: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class Task:
|
||||
id: str
|
||||
user_id: str
|
||||
title: str
|
||||
description: Optional[str]
|
||||
completed: bool
|
||||
created_at: datetime
|
||||
completed_at: Optional[datetime] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Project:
|
||||
id: str
|
||||
user_id: str
|
||||
name: str
|
||||
tasks: List[str] # Task IDs
|
||||
created_at: datetime
|
||||
@@ -1,72 +0,0 @@
|
||||
"""Task management service."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
|
||||
from models import Task
|
||||
from database import db
|
||||
from auth import auth_service, generate_token
|
||||
from validation import validate_task_title
|
||||
|
||||
|
||||
class TaskService:
|
||||
def create_task(
|
||||
self, token: str, title: str, description: Optional[str] = None
|
||||
) -> Optional[Task]:
|
||||
"""Create a new task."""
|
||||
user_id = auth_service.get_user_id(token)
|
||||
if not user_id:
|
||||
return None
|
||||
|
||||
if not validate_task_title(title):
|
||||
return None
|
||||
|
||||
task = Task(
|
||||
id=generate_token(),
|
||||
user_id=user_id,
|
||||
title=title,
|
||||
description=description,
|
||||
completed=False,
|
||||
created_at=datetime.now(),
|
||||
)
|
||||
db.create_task(task)
|
||||
return task
|
||||
|
||||
def get_task(self, token: str, task_id: str) -> Optional[Task]:
|
||||
"""Get a task by ID."""
|
||||
user_id = auth_service.get_user_id(token)
|
||||
if not user_id:
|
||||
return None
|
||||
|
||||
task = db.get_task(task_id)
|
||||
if task and task.user_id == user_id:
|
||||
return task
|
||||
return None
|
||||
|
||||
def get_user_tasks(self, token: str) -> List[Task]:
|
||||
"""Get all tasks for the authenticated user."""
|
||||
user_id = auth_service.get_user_id(token)
|
||||
if not user_id:
|
||||
return []
|
||||
|
||||
return db.get_user_tasks(user_id)
|
||||
|
||||
def complete_task(self, token: str, task_id: str) -> bool:
|
||||
"""Mark a task as completed."""
|
||||
task = self.get_task(token, task_id)
|
||||
if not task:
|
||||
return False
|
||||
|
||||
db.update_task(task_id, completed=True, completed_at=datetime.now())
|
||||
return True
|
||||
|
||||
def delete_task(self, token: str, task_id: str) -> bool:
|
||||
"""Delete a task."""
|
||||
task = self.get_task(token, task_id)
|
||||
if not task:
|
||||
return False
|
||||
|
||||
return db.delete_task(task_id)
|
||||
|
||||
|
||||
task_service = TaskService()
|
||||
@@ -1,27 +0,0 @@
|
||||
"""Validation utilities."""
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def validate_email(email: str) -> bool:
|
||||
"""Validate email format."""
|
||||
pattern = r'^[^\s@]+@[^\s@]+\.[^\s@]+$'
|
||||
return bool(re.match(pattern, email))
|
||||
|
||||
|
||||
def validate_password(password: str) -> bool:
|
||||
"""Validate password strength."""
|
||||
if len(password) < 8:
|
||||
return False
|
||||
if not re.search(r'[A-Z]', password):
|
||||
return False
|
||||
if not re.search(r'[a-z]', password):
|
||||
return False
|
||||
if not re.search(r'[0-9]', password):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def validate_task_title(title: str) -> bool:
|
||||
"""Validate task title."""
|
||||
return bool(title and len(title.strip()) >= 1 and len(title) <= 200)
|
||||
@@ -1,366 +0,0 @@
|
||||
/**
|
||||
* Ground truth definitions for the TypeScript e-commerce fixture
|
||||
*/
|
||||
|
||||
import { FixtureGroundTruth } from '../../types';
|
||||
|
||||
export const typescriptFixture: FixtureGroundTruth = {
|
||||
name: 'typescript-ecommerce',
|
||||
path: '__tests__/evaluation/fixtures/typescript-project',
|
||||
language: 'typescript',
|
||||
totalFiles: 9,
|
||||
approximateTokens: 2500, // Rough estimate
|
||||
|
||||
testCases: [
|
||||
// =========================================================================
|
||||
// Search Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'ts-search-login',
|
||||
description: 'Search for login functionality',
|
||||
query: 'login',
|
||||
type: 'search',
|
||||
expectedSymbols: ['AuthService.login', 'AuthService'],
|
||||
irrelevantSymbols: ['PaymentService', 'OrderService', 'calculateTotal'],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.5,
|
||||
},
|
||||
{
|
||||
id: 'ts-search-validation',
|
||||
description: 'Search for validation functions',
|
||||
query: 'validate',
|
||||
type: 'search',
|
||||
expectedSymbols: ['validateEmail', 'validatePassword', 'validateQuantity', 'validatePrice', 'validateToken'],
|
||||
irrelevantSymbols: ['hashPassword', 'generateToken', 'calculateTotal'],
|
||||
minRecall: 0.6,
|
||||
minPrecision: 0.6,
|
||||
},
|
||||
{
|
||||
id: 'ts-search-payment',
|
||||
description: 'Search for payment processing',
|
||||
query: 'payment process',
|
||||
type: 'search',
|
||||
expectedSymbols: ['PaymentService', 'processPayment', 'payOrder'],
|
||||
irrelevantSymbols: ['AuthService', 'UserService', 'validateEmail'],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.5,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Context Tests (simulating Claude asking for context)
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'ts-context-login-bug',
|
||||
description: 'Build context for fixing a login bug',
|
||||
query: 'fix the bug where login fails with valid credentials',
|
||||
type: 'context',
|
||||
expectedSymbols: [
|
||||
'AuthService.login',
|
||||
'verifyPassword',
|
||||
'db.findUserByEmail',
|
||||
'User',
|
||||
'AuthToken',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'OrderService',
|
||||
'PaymentService',
|
||||
'calculateTotal',
|
||||
'validateQuantity',
|
||||
'Product',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.6,
|
||||
},
|
||||
{
|
||||
id: 'ts-context-order-creation',
|
||||
description: 'Build context for understanding order creation flow',
|
||||
query: 'understand how orders are created and validated',
|
||||
type: 'context',
|
||||
expectedSymbols: [
|
||||
'OrderService.createOrder',
|
||||
'validateQuantity',
|
||||
'db.findProductById',
|
||||
'db.createOrder',
|
||||
'paymentService.calculateTotal',
|
||||
'Order',
|
||||
'OrderItem',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'AuthService.register',
|
||||
'validateEmail',
|
||||
'hashPassword',
|
||||
'UserService',
|
||||
],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.5,
|
||||
},
|
||||
{
|
||||
id: 'ts-context-add-refund',
|
||||
description: 'Build context for adding refund functionality',
|
||||
query: 'add ability to request a refund for paid orders',
|
||||
type: 'context',
|
||||
expectedSymbols: [
|
||||
'PaymentService.refundPayment',
|
||||
'OrderService.cancelOrder',
|
||||
'db.updateOrderStatus',
|
||||
'Order',
|
||||
'PaymentResult',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'AuthService.register',
|
||||
'validateEmail',
|
||||
'hashPassword',
|
||||
'UserService.updateProfile',
|
||||
],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.5,
|
||||
},
|
||||
{
|
||||
id: 'ts-context-user-registration',
|
||||
description: 'Build context for user registration flow',
|
||||
query: 'implement email verification during user registration',
|
||||
type: 'context',
|
||||
expectedSymbols: [
|
||||
'AuthService.register',
|
||||
'validateEmail',
|
||||
'hashPassword',
|
||||
'db.createUser',
|
||||
'db.findUserByEmail',
|
||||
'User',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'OrderService',
|
||||
'PaymentService',
|
||||
'calculateTotal',
|
||||
'Product',
|
||||
],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.6,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Callers Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'ts-callers-validateEmail',
|
||||
description: 'Find all callers of validateEmail',
|
||||
query: 'validateEmail',
|
||||
type: 'callers',
|
||||
targetSymbol: 'validateEmail',
|
||||
expectedSymbols: [
|
||||
'AuthService.register',
|
||||
'UserService.updateProfile',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'OrderService',
|
||||
'PaymentService',
|
||||
'validateQuantity',
|
||||
],
|
||||
minRecall: 1.0, // Should find all callers
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'ts-callers-findUserByEmail',
|
||||
description: 'Find all callers of db.findUserByEmail',
|
||||
query: 'findUserByEmail',
|
||||
type: 'callers',
|
||||
targetSymbol: 'findUserByEmail',
|
||||
expectedSymbols: [
|
||||
'AuthService.register',
|
||||
'AuthService.login',
|
||||
'UserService.getUserByEmail',
|
||||
'UserService.updateProfile',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'OrderService',
|
||||
'PaymentService',
|
||||
'findProductById',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'ts-callers-generateToken',
|
||||
description: 'Find all callers of generateToken',
|
||||
query: 'generateToken',
|
||||
type: 'callers',
|
||||
targetSymbol: 'generateToken',
|
||||
expectedSymbols: [
|
||||
'AuthService.register',
|
||||
'AuthService.createToken',
|
||||
'PaymentService.processPayment',
|
||||
'PaymentService.refundPayment',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validateEmail',
|
||||
'validateQuantity',
|
||||
'calculateTotal',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Callees Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'ts-callees-login',
|
||||
description: 'Find what AuthService.login calls',
|
||||
query: 'login',
|
||||
type: 'callees',
|
||||
targetSymbol: 'login',
|
||||
expectedSymbols: [
|
||||
'db.findUserByEmail',
|
||||
'verifyPassword',
|
||||
'createToken',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'hashPassword',
|
||||
'validateQuantity',
|
||||
'calculateTotal',
|
||||
],
|
||||
minRecall: 1.0,
|
||||
minPrecision: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'ts-callees-createOrder',
|
||||
description: 'Find what OrderService.createOrder calls',
|
||||
query: 'createOrder',
|
||||
type: 'callees',
|
||||
targetSymbol: 'OrderService.createOrder',
|
||||
expectedSymbols: [
|
||||
'authService.validateToken',
|
||||
'validateQuantity',
|
||||
'db.findProductById',
|
||||
'paymentService.calculateTotal',
|
||||
'generateOrderId',
|
||||
'db.createOrder',
|
||||
'db.updateProductStock',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validateEmail',
|
||||
'hashPassword',
|
||||
'refundPayment',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.8,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
// Impact Tests
|
||||
// =========================================================================
|
||||
{
|
||||
id: 'ts-impact-generateToken',
|
||||
description: 'Impact of changing generateToken',
|
||||
query: 'generateToken',
|
||||
type: 'impact',
|
||||
targetSymbol: 'generateToken',
|
||||
expectedSymbols: [
|
||||
// Direct callers
|
||||
'AuthService.register',
|
||||
'AuthService.createToken',
|
||||
'PaymentService.processPayment',
|
||||
'PaymentService.refundPayment',
|
||||
// Indirect (callers of callers)
|
||||
'AuthService.login',
|
||||
'AuthService.refreshToken',
|
||||
'OrderService.payOrder',
|
||||
'OrderService.cancelOrder',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validateQuantity',
|
||||
'validatePrice',
|
||||
'UserService.getUser',
|
||||
],
|
||||
minRecall: 0.7,
|
||||
minPrecision: 0.6,
|
||||
},
|
||||
{
|
||||
id: 'ts-impact-validateToken',
|
||||
description: 'Impact of changing validateToken',
|
||||
query: 'validateToken',
|
||||
type: 'impact',
|
||||
targetSymbol: 'validateToken',
|
||||
expectedSymbols: [
|
||||
// Direct callers
|
||||
'AuthService.refreshToken',
|
||||
'OrderService.createOrder',
|
||||
'OrderService.getOrder',
|
||||
'OrderService.getUserOrders',
|
||||
'OrderService.payOrder',
|
||||
'OrderService.cancelOrder',
|
||||
],
|
||||
irrelevantSymbols: [
|
||||
'validateEmail',
|
||||
'validateQuantity',
|
||||
'hashPassword',
|
||||
'PaymentService.calculateTotal',
|
||||
],
|
||||
minRecall: 0.8,
|
||||
minPrecision: 0.7,
|
||||
},
|
||||
],
|
||||
|
||||
// Known call graph edges for validation
|
||||
callGraph: [
|
||||
// Auth -> Database
|
||||
{ caller: 'AuthService.register', callee: 'db.findUserByEmail' },
|
||||
{ caller: 'AuthService.register', callee: 'db.createUser' },
|
||||
{ caller: 'AuthService.login', callee: 'db.findUserByEmail' },
|
||||
|
||||
// Auth -> Crypto
|
||||
{ caller: 'AuthService.register', callee: 'hashPassword' },
|
||||
{ caller: 'AuthService.register', callee: 'generateToken' },
|
||||
{ caller: 'AuthService.login', callee: 'verifyPassword' },
|
||||
{ caller: 'AuthService.createToken', callee: 'generateToken' },
|
||||
|
||||
// Auth -> Validation
|
||||
{ caller: 'AuthService.register', callee: 'validateEmail' },
|
||||
|
||||
// User -> Database
|
||||
{ caller: 'UserService.getUser', callee: 'db.findUserById' },
|
||||
{ caller: 'UserService.getUserByEmail', callee: 'db.findUserByEmail' },
|
||||
{ caller: 'UserService.updateProfile', callee: 'db.findUserById' },
|
||||
{ caller: 'UserService.updateProfile', callee: 'db.findUserByEmail' },
|
||||
{ caller: 'UserService.updateProfile', callee: 'db.updateUser' },
|
||||
{ caller: 'UserService.deleteUser', callee: 'db.findUserById' },
|
||||
{ caller: 'UserService.deleteUser', callee: 'db.updateUser' },
|
||||
|
||||
// User -> Validation
|
||||
{ caller: 'UserService.updateProfile', callee: 'validateEmail' },
|
||||
|
||||
// Order -> Auth
|
||||
{ caller: 'OrderService.createOrder', callee: 'authService.validateToken' },
|
||||
{ caller: 'OrderService.getOrder', callee: 'authService.validateToken' },
|
||||
{ caller: 'OrderService.getUserOrders', callee: 'authService.validateToken' },
|
||||
|
||||
// Order -> Database
|
||||
{ caller: 'OrderService.createOrder', callee: 'db.findProductById' },
|
||||
{ caller: 'OrderService.createOrder', callee: 'db.createOrder' },
|
||||
{ caller: 'OrderService.createOrder', callee: 'db.updateProductStock' },
|
||||
{ caller: 'OrderService.getOrder', callee: 'db.findOrderById' },
|
||||
{ caller: 'OrderService.getUserOrders', callee: 'db.findOrdersByUserId' },
|
||||
{ caller: 'OrderService.cancelOrder', callee: 'db.updateOrderStatus' },
|
||||
|
||||
// Order -> Payment
|
||||
{ caller: 'OrderService.createOrder', callee: 'paymentService.calculateTotal' },
|
||||
{ caller: 'OrderService.payOrder', callee: 'paymentService.processPayment' },
|
||||
{ caller: 'OrderService.cancelOrder', callee: 'paymentService.refundPayment' },
|
||||
|
||||
// Order -> Validation
|
||||
{ caller: 'OrderService.createOrder', callee: 'validateQuantity' },
|
||||
|
||||
// Order -> Crypto
|
||||
{ caller: 'OrderService.createOrder', callee: 'generateOrderId' },
|
||||
|
||||
// Payment -> Database
|
||||
{ caller: 'PaymentService.processPayment', callee: 'db.findOrderById' },
|
||||
{ caller: 'PaymentService.processPayment', callee: 'db.updateOrderStatus' },
|
||||
{ caller: 'PaymentService.refundPayment', callee: 'db.findOrderById' },
|
||||
{ caller: 'PaymentService.refundPayment', callee: 'db.updateOrderStatus' },
|
||||
|
||||
// Payment -> Crypto
|
||||
{ caller: 'PaymentService.processPayment', callee: 'generateToken' },
|
||||
{ caller: 'PaymentService.refundPayment', callee: 'generateToken' },
|
||||
],
|
||||
};
|
||||
@@ -1,90 +0,0 @@
|
||||
/**
|
||||
* Authentication service
|
||||
*/
|
||||
|
||||
import { User, AuthToken } from './types';
|
||||
import { db } from './database';
|
||||
import { hashPassword, verifyPassword, generateToken } from './utils/crypto';
|
||||
import { validateEmail } from './utils/validation';
|
||||
|
||||
export class AuthService {
|
||||
private tokens: Map<string, AuthToken> = new Map();
|
||||
|
||||
async register(email: string, password: string, name: string): Promise<User> {
|
||||
if (!validateEmail(email)) {
|
||||
throw new Error('Invalid email format');
|
||||
}
|
||||
|
||||
const existing = await db.findUserByEmail(email);
|
||||
if (existing) {
|
||||
throw new Error('Email already registered');
|
||||
}
|
||||
|
||||
const passwordHash = await hashPassword(password);
|
||||
const user: User = {
|
||||
id: generateToken(),
|
||||
email,
|
||||
name,
|
||||
passwordHash,
|
||||
createdAt: new Date(),
|
||||
};
|
||||
|
||||
await db.createUser(user);
|
||||
return user;
|
||||
}
|
||||
|
||||
async login(email: string, password: string): Promise<AuthToken> {
|
||||
const user = await db.findUserByEmail(email);
|
||||
if (!user) {
|
||||
throw new Error('Invalid credentials');
|
||||
}
|
||||
|
||||
const valid = await verifyPassword(password, user.passwordHash);
|
||||
if (!valid) {
|
||||
throw new Error('Invalid credentials');
|
||||
}
|
||||
|
||||
const token = this.createToken(user.id);
|
||||
return token;
|
||||
}
|
||||
|
||||
async logout(token: string): Promise<void> {
|
||||
this.tokens.delete(token);
|
||||
}
|
||||
|
||||
async validateToken(token: string): Promise<string | null> {
|
||||
const authToken = this.tokens.get(token);
|
||||
if (!authToken) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (authToken.expiresAt < new Date()) {
|
||||
this.tokens.delete(token);
|
||||
return null;
|
||||
}
|
||||
|
||||
return authToken.userId;
|
||||
}
|
||||
|
||||
async refreshToken(token: string): Promise<AuthToken | null> {
|
||||
const userId = await this.validateToken(token);
|
||||
if (!userId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
this.tokens.delete(token);
|
||||
return this.createToken(userId);
|
||||
}
|
||||
|
||||
private createToken(userId: string): AuthToken {
|
||||
const token: AuthToken = {
|
||||
token: generateToken(),
|
||||
userId,
|
||||
expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), // 24 hours
|
||||
};
|
||||
this.tokens.set(token.token, token);
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
export const authService = new AuthService();
|
||||
@@ -1,75 +0,0 @@
|
||||
/**
|
||||
* Database abstraction layer
|
||||
*/
|
||||
|
||||
import { User, Product, Order } from './types';
|
||||
|
||||
export class Database {
|
||||
private users: Map<string, User> = new Map();
|
||||
private products: Map<string, Product> = new Map();
|
||||
private orders: Map<string, Order> = new Map();
|
||||
|
||||
async findUserById(id: string): Promise<User | null> {
|
||||
return this.users.get(id) || null;
|
||||
}
|
||||
|
||||
async findUserByEmail(email: string): Promise<User | null> {
|
||||
for (const user of this.users.values()) {
|
||||
if (user.email === email) {
|
||||
return user;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async createUser(user: User): Promise<void> {
|
||||
this.users.set(user.id, user);
|
||||
}
|
||||
|
||||
async updateUser(id: string, updates: Partial<User>): Promise<void> {
|
||||
const user = this.users.get(id);
|
||||
if (user) {
|
||||
this.users.set(id, { ...user, ...updates });
|
||||
}
|
||||
}
|
||||
|
||||
async findProductById(id: string): Promise<Product | null> {
|
||||
return this.products.get(id) || null;
|
||||
}
|
||||
|
||||
async updateProductStock(id: string, quantity: number): Promise<void> {
|
||||
const product = this.products.get(id);
|
||||
if (product) {
|
||||
product.stock -= quantity;
|
||||
this.products.set(id, product);
|
||||
}
|
||||
}
|
||||
|
||||
async createOrder(order: Order): Promise<void> {
|
||||
this.orders.set(order.id, order);
|
||||
}
|
||||
|
||||
async findOrderById(id: string): Promise<Order | null> {
|
||||
return this.orders.get(id) || null;
|
||||
}
|
||||
|
||||
async findOrdersByUserId(userId: string): Promise<Order[]> {
|
||||
const orders: Order[] = [];
|
||||
for (const order of this.orders.values()) {
|
||||
if (order.userId === userId) {
|
||||
orders.push(order);
|
||||
}
|
||||
}
|
||||
return orders;
|
||||
}
|
||||
|
||||
async updateOrderStatus(id: string, status: Order['status']): Promise<void> {
|
||||
const order = this.orders.get(id);
|
||||
if (order) {
|
||||
order.status = status;
|
||||
this.orders.set(id, order);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const db = new Database();
|
||||
@@ -1,11 +0,0 @@
|
||||
/**
|
||||
* E-commerce application entry point
|
||||
*/
|
||||
|
||||
export { authService, AuthService } from './auth';
|
||||
export { userService, UserService } from './user';
|
||||
export { orderService, OrderService } from './order';
|
||||
export { paymentService, PaymentService } from './payment';
|
||||
export { db, Database } from './database';
|
||||
|
||||
export * from './types';
|
||||
@@ -1,115 +0,0 @@
|
||||
/**
|
||||
* Order management service
|
||||
*/
|
||||
|
||||
import { Order, OrderItem, Product } from './types';
|
||||
import { db } from './database';
|
||||
import { paymentService } from './payment';
|
||||
import { authService } from './auth';
|
||||
import { generateOrderId } from './utils/crypto';
|
||||
import { validateQuantity } from './utils/validation';
|
||||
|
||||
export class OrderService {
|
||||
async createOrder(token: string, items: OrderItem[]): Promise<Order> {
|
||||
const userId = await authService.validateToken(token);
|
||||
if (!userId) {
|
||||
throw new Error('Invalid or expired token');
|
||||
}
|
||||
|
||||
// Validate items
|
||||
for (const item of items) {
|
||||
if (!validateQuantity(item.quantity)) {
|
||||
throw new Error(`Invalid quantity for product ${item.productId}`);
|
||||
}
|
||||
|
||||
const product = await db.findProductById(item.productId);
|
||||
if (!product) {
|
||||
throw new Error(`Product not found: ${item.productId}`);
|
||||
}
|
||||
|
||||
if (product.stock < item.quantity) {
|
||||
throw new Error(`Insufficient stock for product ${item.productId}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate total
|
||||
const total = paymentService.calculateTotal(items);
|
||||
|
||||
// Create order
|
||||
const order: Order = {
|
||||
id: generateOrderId(),
|
||||
userId,
|
||||
items,
|
||||
total,
|
||||
status: 'pending',
|
||||
createdAt: new Date(),
|
||||
};
|
||||
|
||||
await db.createOrder(order);
|
||||
|
||||
// Update stock
|
||||
for (const item of items) {
|
||||
await db.updateProductStock(item.productId, item.quantity);
|
||||
}
|
||||
|
||||
return order;
|
||||
}
|
||||
|
||||
async getOrder(token: string, orderId: string): Promise<Order | null> {
|
||||
const userId = await authService.validateToken(token);
|
||||
if (!userId) {
|
||||
throw new Error('Invalid or expired token');
|
||||
}
|
||||
|
||||
const order = await db.findOrderById(orderId);
|
||||
if (!order || order.userId !== userId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return order;
|
||||
}
|
||||
|
||||
async getUserOrders(token: string): Promise<Order[]> {
|
||||
const userId = await authService.validateToken(token);
|
||||
if (!userId) {
|
||||
throw new Error('Invalid or expired token');
|
||||
}
|
||||
|
||||
return db.findOrdersByUserId(userId);
|
||||
}
|
||||
|
||||
async payOrder(token: string, orderId: string): Promise<boolean> {
|
||||
const order = await this.getOrder(token, orderId);
|
||||
if (!order) {
|
||||
throw new Error('Order not found');
|
||||
}
|
||||
|
||||
if (order.status !== 'pending') {
|
||||
throw new Error('Order already processed');
|
||||
}
|
||||
|
||||
const result = await paymentService.processPayment(orderId, order.total);
|
||||
return result.success;
|
||||
}
|
||||
|
||||
async cancelOrder(token: string, orderId: string): Promise<boolean> {
|
||||
const order = await this.getOrder(token, orderId);
|
||||
if (!order) {
|
||||
throw new Error('Order not found');
|
||||
}
|
||||
|
||||
if (order.status === 'shipped' || order.status === 'delivered') {
|
||||
throw new Error('Cannot cancel shipped or delivered orders');
|
||||
}
|
||||
|
||||
if (order.status === 'paid') {
|
||||
const refund = await paymentService.refundPayment(orderId);
|
||||
return refund.success;
|
||||
}
|
||||
|
||||
await db.updateOrderStatus(orderId, 'cancelled');
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export const orderService = new OrderService();
|
||||
@@ -1,60 +0,0 @@
|
||||
/**
|
||||
* Payment processing service
|
||||
*/
|
||||
|
||||
import { PaymentResult, Order } from './types';
|
||||
import { db } from './database';
|
||||
import { generateToken } from './utils/crypto';
|
||||
|
||||
export class PaymentService {
|
||||
async processPayment(orderId: string, amount: number): Promise<PaymentResult> {
|
||||
const order = await db.findOrderById(orderId);
|
||||
if (!order) {
|
||||
return { success: false, error: 'Order not found' };
|
||||
}
|
||||
|
||||
if (order.status !== 'pending') {
|
||||
return { success: false, error: 'Order already processed' };
|
||||
}
|
||||
|
||||
if (order.total !== amount) {
|
||||
return { success: false, error: 'Amount mismatch' };
|
||||
}
|
||||
|
||||
// Simulate payment processing
|
||||
const success = Math.random() > 0.1; // 90% success rate
|
||||
|
||||
if (success) {
|
||||
await db.updateOrderStatus(orderId, 'paid');
|
||||
return {
|
||||
success: true,
|
||||
transactionId: generateToken(),
|
||||
};
|
||||
}
|
||||
|
||||
return { success: false, error: 'Payment declined' };
|
||||
}
|
||||
|
||||
async refundPayment(orderId: string): Promise<PaymentResult> {
|
||||
const order = await db.findOrderById(orderId);
|
||||
if (!order) {
|
||||
return { success: false, error: 'Order not found' };
|
||||
}
|
||||
|
||||
if (order.status !== 'paid') {
|
||||
return { success: false, error: 'Order not eligible for refund' };
|
||||
}
|
||||
|
||||
await db.updateOrderStatus(orderId, 'cancelled');
|
||||
return {
|
||||
success: true,
|
||||
transactionId: generateToken(),
|
||||
};
|
||||
}
|
||||
|
||||
calculateTotal(items: { price: number; quantity: number }[]): number {
|
||||
return items.reduce((sum, item) => sum + item.price * item.quantity, 0);
|
||||
}
|
||||
}
|
||||
|
||||
export const paymentService = new PaymentService();
|
||||
@@ -1,47 +0,0 @@
|
||||
/**
|
||||
* Core types for the e-commerce application
|
||||
*/
|
||||
|
||||
export interface User {
|
||||
id: string;
|
||||
email: string;
|
||||
name: string;
|
||||
passwordHash: string;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface Product {
|
||||
id: string;
|
||||
name: string;
|
||||
price: number;
|
||||
stock: number;
|
||||
}
|
||||
|
||||
export interface OrderItem {
|
||||
productId: string;
|
||||
quantity: number;
|
||||
price: number;
|
||||
}
|
||||
|
||||
export interface Order {
|
||||
id: string;
|
||||
userId: string;
|
||||
items: OrderItem[];
|
||||
total: number;
|
||||
status: OrderStatus;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export type OrderStatus = 'pending' | 'paid' | 'shipped' | 'delivered' | 'cancelled';
|
||||
|
||||
export interface PaymentResult {
|
||||
success: boolean;
|
||||
transactionId?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface AuthToken {
|
||||
token: string;
|
||||
userId: string;
|
||||
expiresAt: Date;
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
/**
|
||||
* User management service
|
||||
*/
|
||||
|
||||
import { User } from './types';
|
||||
import { db } from './database';
|
||||
import { validateEmail } from './utils/validation';
|
||||
|
||||
export class UserService {
|
||||
async getUser(id: string): Promise<User | null> {
|
||||
return db.findUserById(id);
|
||||
}
|
||||
|
||||
async getUserByEmail(email: string): Promise<User | null> {
|
||||
return db.findUserByEmail(email);
|
||||
}
|
||||
|
||||
async updateProfile(userId: string, updates: { name?: string; email?: string }): Promise<User> {
|
||||
const user = await db.findUserById(userId);
|
||||
if (!user) {
|
||||
throw new Error('User not found');
|
||||
}
|
||||
|
||||
if (updates.email && updates.email !== user.email) {
|
||||
if (!validateEmail(updates.email)) {
|
||||
throw new Error('Invalid email format');
|
||||
}
|
||||
|
||||
const existing = await db.findUserByEmail(updates.email);
|
||||
if (existing) {
|
||||
throw new Error('Email already in use');
|
||||
}
|
||||
}
|
||||
|
||||
await db.updateUser(userId, updates);
|
||||
return { ...user, ...updates };
|
||||
}
|
||||
|
||||
async deleteUser(userId: string): Promise<void> {
|
||||
const user = await db.findUserById(userId);
|
||||
if (!user) {
|
||||
throw new Error('User not found');
|
||||
}
|
||||
|
||||
// In a real app, we'd also delete orders, etc.
|
||||
await db.updateUser(userId, { email: `deleted_${userId}@deleted.com` });
|
||||
}
|
||||
}
|
||||
|
||||
export const userService = new UserService();
|
||||
@@ -1,21 +0,0 @@
|
||||
/**
|
||||
* Cryptographic utilities
|
||||
*/
|
||||
|
||||
export async function hashPassword(password: string): Promise<string> {
|
||||
// Simulated password hashing
|
||||
return `hashed_${password}_${Date.now()}`;
|
||||
}
|
||||
|
||||
export async function verifyPassword(password: string, hash: string): Promise<boolean> {
|
||||
// Simulated password verification
|
||||
return hash.startsWith(`hashed_${password}_`);
|
||||
}
|
||||
|
||||
export function generateToken(): string {
|
||||
return Math.random().toString(36).substring(2) + Date.now().toString(36);
|
||||
}
|
||||
|
||||
export function generateOrderId(): string {
|
||||
return `ORD-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
/**
|
||||
* Validation utilities
|
||||
*/
|
||||
|
||||
export function validateEmail(email: string): boolean {
|
||||
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
||||
return emailRegex.test(email);
|
||||
}
|
||||
|
||||
export function validatePassword(password: string): { valid: boolean; errors: string[] } {
|
||||
const errors: string[] = [];
|
||||
|
||||
if (password.length < 8) {
|
||||
errors.push('Password must be at least 8 characters');
|
||||
}
|
||||
if (!/[A-Z]/.test(password)) {
|
||||
errors.push('Password must contain an uppercase letter');
|
||||
}
|
||||
if (!/[a-z]/.test(password)) {
|
||||
errors.push('Password must contain a lowercase letter');
|
||||
}
|
||||
if (!/[0-9]/.test(password)) {
|
||||
errors.push('Password must contain a number');
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
export function validateQuantity(quantity: number): boolean {
|
||||
return Number.isInteger(quantity) && quantity > 0;
|
||||
}
|
||||
|
||||
export function validatePrice(price: number): boolean {
|
||||
return typeof price === 'number' && price >= 0;
|
||||
}
|
||||
@@ -1,374 +0,0 @@
|
||||
/**
|
||||
* Evaluation Runner
|
||||
*
|
||||
* Runs test cases against CodeGraph fixtures and measures precision/recall.
|
||||
*/
|
||||
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import CodeGraph from '../../src/index';
|
||||
import type { Node, SearchResult, NodeKind } from '../../src/types';
|
||||
import type {
|
||||
TestCase,
|
||||
TestCaseResult,
|
||||
FixtureGroundTruth,
|
||||
FixtureEvaluationResult,
|
||||
EvaluationSummary,
|
||||
} from './types';
|
||||
|
||||
// Import fixtures
|
||||
import { typescriptFixture } from './fixtures/typescript-project/ground-truth';
|
||||
import { pythonFixture } from './fixtures/python-project/ground-truth';
|
||||
|
||||
/**
|
||||
* Simple token counter (approximation using word count * 1.3)
|
||||
*/
|
||||
function countTokens(text: string): number {
|
||||
const words = text.split(/\s+/).filter(w => w.length > 0);
|
||||
return Math.ceil(words.length * 1.3);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract symbol names from CodeGraph results
|
||||
*/
|
||||
function extractSymbolNames(nodes: Node[]): Set<string> {
|
||||
const names = new Set<string>();
|
||||
for (const node of nodes) {
|
||||
// Add the simple name
|
||||
names.add(node.name);
|
||||
|
||||
// Add qualified name if we have parent info (Class.method format)
|
||||
// This is a simplification - real implementation would use containment edges
|
||||
if (node.kind === 'method' || node.kind === 'function') {
|
||||
// Try to infer class from file path or other context
|
||||
const fileName = path.basename(node.filePath, path.extname(node.filePath));
|
||||
names.add(`${fileName}.${node.name}`);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize symbol name for comparison
|
||||
*/
|
||||
function normalizeSymbol(symbol: string): string {
|
||||
// Remove common prefixes and normalize
|
||||
return symbol
|
||||
.replace(/^(db\.|authService\.|paymentService\.|auth_service\.|task_service\.)/, '')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a symbol matches any in a set (with fuzzy matching)
|
||||
*/
|
||||
function symbolMatches(symbol: string, candidates: Set<string>): boolean {
|
||||
const normalized = normalizeSymbol(symbol);
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const normalizedCandidate = normalizeSymbol(candidate);
|
||||
|
||||
// Exact match
|
||||
if (normalized === normalizedCandidate) return true;
|
||||
|
||||
// Partial match (e.g., "login" matches "AuthService.login")
|
||||
if (normalizedCandidate.endsWith(`.${normalized}`)) return true;
|
||||
if (normalized.endsWith(`.${normalizedCandidate}`)) return true;
|
||||
|
||||
// Simple name match
|
||||
const simpleName = normalized.split('.').pop();
|
||||
const simpleCandidateName = normalizedCandidate.split('.').pop();
|
||||
if (simpleName === simpleCandidateName) return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a single test case
|
||||
*/
|
||||
async function runTestCase(
|
||||
cg: CodeGraph,
|
||||
testCase: TestCase,
|
||||
fixtureTokens: number
|
||||
): Promise<TestCaseResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
let retrievedNodes: Node[] = [];
|
||||
let contextText = '';
|
||||
|
||||
try {
|
||||
switch (testCase.type) {
|
||||
case 'search': {
|
||||
const results = cg.searchNodes(testCase.query, { limit: 20 });
|
||||
retrievedNodes = results.map(r => r.node);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'context': {
|
||||
const context = await cg.buildContext(testCase.query, {
|
||||
maxNodes: 30,
|
||||
includeCode: true,
|
||||
format: 'markdown',
|
||||
});
|
||||
contextText = typeof context === 'string' ? context : '';
|
||||
|
||||
// Also get the nodes that were used to build context
|
||||
const results = cg.searchNodes(testCase.query, { limit: 30 });
|
||||
retrievedNodes = results.map(r => r.node);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'callers': {
|
||||
if (testCase.targetSymbol) {
|
||||
const results = cg.searchNodes(testCase.targetSymbol, { limit: 1 });
|
||||
if (results.length > 0 && results[0]) {
|
||||
const callers = cg.getCallers(results[0].node.id);
|
||||
retrievedNodes = callers.map(c => c.node);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'callees': {
|
||||
if (testCase.targetSymbol) {
|
||||
const results = cg.searchNodes(testCase.targetSymbol, { limit: 1 });
|
||||
if (results.length > 0 && results[0]) {
|
||||
const callees = cg.getCallees(results[0].node.id);
|
||||
retrievedNodes = callees.map(c => c.node);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'impact': {
|
||||
if (testCase.targetSymbol) {
|
||||
const results = cg.searchNodes(testCase.targetSymbol, { limit: 1 });
|
||||
if (results.length > 0 && results[0]) {
|
||||
const impact = cg.getImpactRadius(results[0].node.id, 2);
|
||||
retrievedNodes = Array.from(impact.nodes.values());
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error running test case ${testCase.id}:`, err);
|
||||
}
|
||||
|
||||
const executionTimeMs = Date.now() - startTime;
|
||||
|
||||
// Extract retrieved symbol names
|
||||
const retrievedSymbols = extractSymbolNames(retrievedNodes);
|
||||
|
||||
// Calculate metrics
|
||||
const expectedSet = new Set(testCase.expectedSymbols.map(s => normalizeSymbol(s)));
|
||||
const irrelevantSet = new Set(testCase.irrelevantSymbols.map(s => normalizeSymbol(s)));
|
||||
|
||||
const truePositives: string[] = [];
|
||||
const falsePositives: string[] = [];
|
||||
|
||||
for (const symbol of retrievedSymbols) {
|
||||
const normalized = normalizeSymbol(symbol);
|
||||
|
||||
if (symbolMatches(symbol, new Set(testCase.expectedSymbols))) {
|
||||
truePositives.push(symbol);
|
||||
} else if (symbolMatches(symbol, new Set(testCase.irrelevantSymbols))) {
|
||||
falsePositives.push(symbol);
|
||||
}
|
||||
// Symbols not in either list are ignored (neutral)
|
||||
}
|
||||
|
||||
// Find false negatives (expected but not retrieved)
|
||||
const falseNegatives: string[] = [];
|
||||
for (const expected of testCase.expectedSymbols) {
|
||||
if (!symbolMatches(expected, retrievedSymbols)) {
|
||||
falseNegatives.push(expected);
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate precision and recall
|
||||
const totalRetrieved = truePositives.length + falsePositives.length;
|
||||
const precision = totalRetrieved > 0 ? truePositives.length / totalRetrieved : 0;
|
||||
|
||||
const totalRelevant = testCase.expectedSymbols.length;
|
||||
const recall = totalRelevant > 0 ? truePositives.length / totalRelevant : 0;
|
||||
|
||||
const f1Score = precision + recall > 0
|
||||
? 2 * (precision * recall) / (precision + recall)
|
||||
: 0;
|
||||
|
||||
// Count context tokens
|
||||
const contextTokens = contextText
|
||||
? countTokens(contextText)
|
||||
: retrievedNodes.reduce((sum, node) => {
|
||||
// Estimate tokens from node info
|
||||
return sum + countTokens(node.name + ' ' + (node.signature || ''));
|
||||
}, 0);
|
||||
|
||||
// Determine if test passed
|
||||
const meetsRecall = !testCase.minRecall || recall >= testCase.minRecall;
|
||||
const meetsPrecision = !testCase.minPrecision || precision >= testCase.minPrecision;
|
||||
const passed = meetsRecall && meetsPrecision;
|
||||
|
||||
return {
|
||||
testCaseId: testCase.id,
|
||||
passed,
|
||||
precision,
|
||||
recall,
|
||||
f1Score,
|
||||
truePositives,
|
||||
falsePositives,
|
||||
falseNegatives,
|
||||
contextTokens,
|
||||
executionTimeMs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run evaluation on a single fixture
|
||||
*/
|
||||
async function evaluateFixture(
|
||||
fixture: FixtureGroundTruth
|
||||
): Promise<FixtureEvaluationResult> {
|
||||
const fixturePath = path.resolve(process.cwd(), fixture.path);
|
||||
const startTime = Date.now();
|
||||
|
||||
console.log(`\nEvaluating fixture: ${fixture.name}`);
|
||||
console.log(` Path: ${fixturePath}`);
|
||||
|
||||
// Initialize CodeGraph for this fixture
|
||||
let cg: CodeGraph;
|
||||
|
||||
if (CodeGraph.isInitialized(fixturePath)) {
|
||||
console.log(' Opening existing index...');
|
||||
cg = await CodeGraph.open(fixturePath);
|
||||
} else {
|
||||
console.log(' Initializing and indexing...');
|
||||
cg = await CodeGraph.init(fixturePath, { index: true });
|
||||
}
|
||||
|
||||
const stats = cg.getStats();
|
||||
console.log(` Indexed ${stats.fileCount} files, ${stats.nodeCount} nodes`);
|
||||
|
||||
// Run all test cases
|
||||
const testCaseResults: TestCaseResult[] = [];
|
||||
|
||||
for (const testCase of fixture.testCases) {
|
||||
console.log(` Running: ${testCase.id}...`);
|
||||
const result = await runTestCase(cg, testCase, fixture.approximateTokens);
|
||||
testCaseResults.push(result);
|
||||
|
||||
const status = result.passed ? '✓' : '✗';
|
||||
console.log(` ${status} P=${(result.precision * 100).toFixed(0)}% R=${(result.recall * 100).toFixed(0)}% F1=${(result.f1Score * 100).toFixed(0)}%`);
|
||||
}
|
||||
|
||||
// Close CodeGraph
|
||||
cg.destroy();
|
||||
|
||||
// Calculate aggregate metrics
|
||||
const totalTimeMs = Date.now() - startTime;
|
||||
const passedTestCases = testCaseResults.filter(r => r.passed).length;
|
||||
|
||||
const averagePrecision = testCaseResults.reduce((sum, r) => sum + r.precision, 0) / testCaseResults.length;
|
||||
const averageRecall = testCaseResults.reduce((sum, r) => sum + r.recall, 0) / testCaseResults.length;
|
||||
const averageF1Score = testCaseResults.reduce((sum, r) => sum + r.f1Score, 0) / testCaseResults.length;
|
||||
const averageContextTokens = testCaseResults.reduce((sum, r) => sum + r.contextTokens, 0) / testCaseResults.length;
|
||||
|
||||
const tokenReductionPercent = fixture.approximateTokens > 0
|
||||
? ((fixture.approximateTokens - averageContextTokens) / fixture.approximateTokens) * 100
|
||||
: 0;
|
||||
|
||||
return {
|
||||
fixtureName: fixture.name,
|
||||
totalTestCases: testCaseResults.length,
|
||||
passedTestCases,
|
||||
averagePrecision,
|
||||
averageRecall,
|
||||
averageF1Score,
|
||||
fullCodebaseTokens: fixture.approximateTokens,
|
||||
averageContextTokens,
|
||||
tokenReductionPercent,
|
||||
testCaseResults,
|
||||
totalTimeMs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run full evaluation across all fixtures
|
||||
*/
|
||||
export async function runEvaluation(): Promise<EvaluationSummary> {
|
||||
console.log('╔════════════════════════════════════════════════════════════════╗');
|
||||
console.log('║ CodeGraph Evaluation Suite ║');
|
||||
console.log('╚════════════════════════════════════════════════════════════════╝');
|
||||
|
||||
const fixtures: FixtureGroundTruth[] = [
|
||||
typescriptFixture,
|
||||
pythonFixture,
|
||||
];
|
||||
|
||||
const fixtureResults: FixtureEvaluationResult[] = [];
|
||||
|
||||
for (const fixture of fixtures) {
|
||||
const result = await evaluateFixture(fixture);
|
||||
fixtureResults.push(result);
|
||||
}
|
||||
|
||||
// Calculate overall metrics
|
||||
const totalTests = fixtureResults.reduce((sum, r) => sum + r.totalTestCases, 0);
|
||||
const totalPassed = fixtureResults.reduce((sum, r) => sum + r.passedTestCases, 0);
|
||||
|
||||
const overallPrecision = fixtureResults.reduce((sum, r) => sum + r.averagePrecision, 0) / fixtureResults.length;
|
||||
const overallRecall = fixtureResults.reduce((sum, r) => sum + r.averageRecall, 0) / fixtureResults.length;
|
||||
const overallF1Score = fixtureResults.reduce((sum, r) => sum + r.averageF1Score, 0) / fixtureResults.length;
|
||||
const overallTokenReduction = fixtureResults.reduce((sum, r) => sum + r.tokenReductionPercent, 0) / fixtureResults.length;
|
||||
|
||||
// Print summary
|
||||
console.log('\n╔════════════════════════════════════════════════════════════════╗');
|
||||
console.log('║ EVALUATION SUMMARY ║');
|
||||
console.log('╚════════════════════════════════════════════════════════════════╝');
|
||||
|
||||
console.log(`\nTest Results: ${totalPassed}/${totalTests} passed`);
|
||||
console.log(`\nOverall Metrics:`);
|
||||
console.log(` Precision: ${(overallPrecision * 100).toFixed(1)}%`);
|
||||
console.log(` Recall: ${(overallRecall * 100).toFixed(1)}%`);
|
||||
console.log(` F1 Score: ${(overallF1Score * 100).toFixed(1)}%`);
|
||||
console.log(` Token Reduction: ${overallTokenReduction.toFixed(1)}%`);
|
||||
|
||||
console.log('\nPer-Fixture Results:');
|
||||
for (const result of fixtureResults) {
|
||||
console.log(` ${result.fixtureName}:`);
|
||||
console.log(` Tests: ${result.passedTestCases}/${result.totalTestCases} passed`);
|
||||
console.log(` P=${(result.averagePrecision * 100).toFixed(0)}% R=${(result.averageRecall * 100).toFixed(0)}% F1=${(result.averageF1Score * 100).toFixed(0)}%`);
|
||||
}
|
||||
|
||||
const summary: EvaluationSummary = {
|
||||
timestamp: new Date(),
|
||||
version: '0.1.0',
|
||||
fixtureResults,
|
||||
overallPrecision,
|
||||
overallRecall,
|
||||
overallF1Score,
|
||||
overallTokenReduction,
|
||||
};
|
||||
|
||||
// Save results to file
|
||||
const resultsPath = path.join(__dirname, 'results', `eval-${Date.now()}.json`);
|
||||
const resultsDir = path.dirname(resultsPath);
|
||||
if (!fs.existsSync(resultsDir)) {
|
||||
fs.mkdirSync(resultsDir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(resultsPath, JSON.stringify(summary, null, 2));
|
||||
console.log(`\nResults saved to: ${resultsPath}`);
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
runEvaluation()
|
||||
.then(() => process.exit(0))
|
||||
.catch(err => {
|
||||
console.error('Evaluation failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
/**
|
||||
* Evaluation Framework Types
|
||||
*/
|
||||
|
||||
/**
|
||||
* A test case with expected ground truth
|
||||
*/
|
||||
export interface TestCase {
|
||||
/** Unique identifier for this test case */
|
||||
id: string;
|
||||
|
||||
/** Human-readable description */
|
||||
description: string;
|
||||
|
||||
/** The query/task to test */
|
||||
query: string;
|
||||
|
||||
/** Type of operation being tested */
|
||||
type: 'search' | 'callers' | 'callees' | 'impact' | 'context';
|
||||
|
||||
/** For callers/callees/impact: the symbol to analyze */
|
||||
targetSymbol?: string;
|
||||
|
||||
/** Symbols that MUST be in the results (for recall) */
|
||||
expectedSymbols: string[];
|
||||
|
||||
/** Symbols that should NOT be in the results (for precision) */
|
||||
irrelevantSymbols: string[];
|
||||
|
||||
/** Minimum acceptable recall (0-1) */
|
||||
minRecall?: number;
|
||||
|
||||
/** Minimum acceptable precision (0-1) */
|
||||
minPrecision?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ground truth for a test fixture
|
||||
*/
|
||||
export interface FixtureGroundTruth {
|
||||
/** Fixture name */
|
||||
name: string;
|
||||
|
||||
/** Path to the fixture directory */
|
||||
path: string;
|
||||
|
||||
/** Language of the fixture */
|
||||
language: string;
|
||||
|
||||
/** Total files in the fixture */
|
||||
totalFiles: number;
|
||||
|
||||
/** Approximate total tokens in the fixture */
|
||||
approximateTokens: number;
|
||||
|
||||
/** Test cases for this fixture */
|
||||
testCases: TestCase[];
|
||||
|
||||
/** Known call graph edges for validation */
|
||||
callGraph: {
|
||||
caller: string; // qualified name
|
||||
callee: string; // qualified name
|
||||
}[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Results from evaluating a single test case
|
||||
*/
|
||||
export interface TestCaseResult {
|
||||
/** Test case ID */
|
||||
testCaseId: string;
|
||||
|
||||
/** Whether the test passed */
|
||||
passed: boolean;
|
||||
|
||||
/** Precision: relevant retrieved / total retrieved */
|
||||
precision: number;
|
||||
|
||||
/** Recall: relevant retrieved / total relevant */
|
||||
recall: number;
|
||||
|
||||
/** F1 score: 2 * (precision * recall) / (precision + recall) */
|
||||
f1Score: number;
|
||||
|
||||
/** Symbols that were correctly retrieved */
|
||||
truePositives: string[];
|
||||
|
||||
/** Irrelevant symbols that were incorrectly retrieved */
|
||||
falsePositives: string[];
|
||||
|
||||
/** Expected symbols that were missed */
|
||||
falseNegatives: string[];
|
||||
|
||||
/** Tokens in the retrieved context */
|
||||
contextTokens: number;
|
||||
|
||||
/** Execution time in ms */
|
||||
executionTimeMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Results from evaluating a fixture
|
||||
*/
|
||||
export interface FixtureEvaluationResult {
|
||||
/** Fixture name */
|
||||
fixtureName: string;
|
||||
|
||||
/** Total test cases */
|
||||
totalTestCases: number;
|
||||
|
||||
/** Passed test cases */
|
||||
passedTestCases: number;
|
||||
|
||||
/** Average precision across all tests */
|
||||
averagePrecision: number;
|
||||
|
||||
/** Average recall across all tests */
|
||||
averageRecall: number;
|
||||
|
||||
/** Average F1 score */
|
||||
averageF1Score: number;
|
||||
|
||||
/** Total tokens in the full codebase */
|
||||
fullCodebaseTokens: number;
|
||||
|
||||
/** Average tokens in retrieved context */
|
||||
averageContextTokens: number;
|
||||
|
||||
/** Token reduction percentage */
|
||||
tokenReductionPercent: number;
|
||||
|
||||
/** Individual test case results */
|
||||
testCaseResults: TestCaseResult[];
|
||||
|
||||
/** Total evaluation time in ms */
|
||||
totalTimeMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Overall evaluation summary
|
||||
*/
|
||||
export interface EvaluationSummary {
|
||||
/** Timestamp of the evaluation */
|
||||
timestamp: Date;
|
||||
|
||||
/** CodeGraph version */
|
||||
version: string;
|
||||
|
||||
/** Results per fixture */
|
||||
fixtureResults: FixtureEvaluationResult[];
|
||||
|
||||
/** Overall average precision */
|
||||
overallPrecision: number;
|
||||
|
||||
/** Overall average recall */
|
||||
overallRecall: number;
|
||||
|
||||
/** Overall average F1 */
|
||||
overallF1Score: number;
|
||||
|
||||
/** Overall token reduction */
|
||||
overallTokenReduction: number;
|
||||
}
|
||||
Reference in New Issue
Block a user