diff --git a/CLAUDE.md b/CLAUDE.md index 3f78f22..42fde5e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,10 +58,6 @@ src/ │ ├── index.ts # GraphQueryManager │ ├── traversal.ts # GraphTraverser (BFS/DFS, impact radius) │ └── queries.ts # High-level graph queries -├── vectors/ # Semantic search with embeddings -│ ├── index.ts # VectorManager -│ ├── embedder.ts # ONNX runtime + model loading -│ └── search.ts # Similarity search ├── context/ # Context building for AI assistants │ ├── index.ts # ContextBuilder │ └── formatter.ts # Markdown/JSON output formatting @@ -83,14 +79,12 @@ src/ ### Key Classes -- **CodeGraph** (`src/index.ts`): Main entry point. Lifecycle methods (`init`, `open`, `close`), indexing (`indexAll`, `sync`), graph queries (`traverse`, `getCallGraph`, `getImpactRadius`), semantic search (`semanticSearch`, `findSimilar`), context building (`buildContext`) +- **CodeGraph** (`src/index.ts`): Main entry point. Lifecycle methods (`init`, `open`, `close`), indexing (`indexAll`, `sync`), graph queries (`traverse`, `getCallGraph`, `getImpactRadius`), context building (`buildContext`) - **ExtractionOrchestrator** (`src/extraction/index.ts`): Coordinates file scanning, parsing, and storing. Uses tree-sitter native bindings for each supported language - **GraphTraverser** (`src/graph/traversal.ts`): BFS/DFS traversal, call graph construction, impact radius calculation, path finding -- **VectorManager** (`src/vectors/manager.ts`): Manages embeddings using `@xenova/transformers` for ONNX inference. Stores vectors in SQLite BLOB format - - **ReferenceResolver** (`src/resolution/index.ts`): Resolves unresolved references after full indexing using framework patterns, import resolution, and name matching ### Database Schema @@ -100,7 +94,6 @@ SQLite database with: - `edges`: Relationships (calls, imports, extends, contains, etc.) - `files`: Tracked source files with content hashes - `unresolved_refs`: References pending resolution -- `vectors`: Embeddings stored as BLOBs - `nodes_fts`: FTS5 virtual table for full-text search ### Supported Languages @@ -153,7 +146,6 @@ Tests are in `__tests__/` directory with files mirroring the module structure: - `extraction.test.ts` - Tree-sitter parsing for all languages - `resolution.test.ts` - Reference resolution - `graph.test.ts` - Traversal and graph queries -- `vectors.test.ts` - Embedding and semantic search - `context.test.ts` - Context building - `sync.test.ts` - Incremental updates and git hooks diff --git a/__tests__/foundation.test.ts b/__tests__/foundation.test.ts index 1dc5559..9ee437d 100644 --- a/__tests__/foundation.test.ts +++ b/__tests__/foundation.test.ts @@ -275,18 +275,6 @@ describe('CodeGraph Foundation', () => { cg.close(); }); - it('should require embedding initialization for semantic search', async () => { - const cg = CodeGraph.initSync(tempDir); - - // Semantic search requires embeddings to be initialized first - await expect(cg.semanticSearch('test')).rejects.toThrow(/not initialized/i); - await expect(cg.findSimilar('test')).rejects.toThrow(/not initialized/i); - - // Check embedding status - expect(cg.isEmbeddingsInitialized()).toBe(false); - - cg.close(); - }); }); }); diff --git a/__tests__/vectors.test.ts b/__tests__/vectors.test.ts deleted file mode 100644 index 449d8de..0000000 --- a/__tests__/vectors.test.ts +++ /dev/null @@ -1,303 +0,0 @@ -/** - * Vector Embedding Tests - * - * Tests for vector embedding and semantic search functionality. - * Note: Full embedding tests require the model to be downloaded, - * which can take time on first run. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import * as fs from 'fs'; -import * as path from 'path'; -import * as os from 'os'; -import CodeGraph from '../src/index'; -import { TextEmbedder } from '../src/vectors/embedder'; -import { VectorSearchManager, createVectorSearch } from '../src/vectors/search'; -import { DatabaseConnection } from '../src/db'; - -describe('Vector Embeddings', () => { - describe('TextEmbedder', () => { - describe('createNodeText', () => { - it('should create text representation from node', () => { - const node = { - name: 'processPayment', - kind: 'function', - qualifiedName: 'PaymentService.processPayment', - signature: '(amount: number) => Promise', - docstring: 'Process a payment and return a receipt.', - filePath: 'src/services/payment.ts', - }; - - const text = TextEmbedder.createNodeText(node); - - expect(text).toContain('function: processPayment'); - expect(text).toContain('path: PaymentService.processPayment'); - expect(text).toContain('file: src/services/payment.ts'); - expect(text).toContain('signature: (amount: number) => Promise'); - expect(text).toContain('documentation: Process a payment'); - }); - - it('should handle minimal node data', () => { - const node = { - name: 'helper', - kind: 'function', - filePath: 'src/utils.ts', - }; - - const text = TextEmbedder.createNodeText(node); - - expect(text).toContain('function: helper'); - expect(text).toContain('file: src/utils.ts'); - expect(text).not.toContain('signature:'); - expect(text).not.toContain('documentation:'); - }); - }); - - describe('cosineSimilarity', () => { - it('should compute similarity between identical vectors', () => { - const vec = new Float32Array([0.1, 0.2, 0.3, 0.4, 0.5]); - const similarity = TextEmbedder.cosineSimilarity(vec, vec); - - expect(similarity).toBeCloseTo(1.0, 5); - }); - - it('should compute similarity between orthogonal vectors', () => { - const vec1 = new Float32Array([1, 0, 0]); - const vec2 = new Float32Array([0, 1, 0]); - const similarity = TextEmbedder.cosineSimilarity(vec1, vec2); - - expect(similarity).toBeCloseTo(0.0, 5); - }); - - it('should compute similarity between opposite vectors', () => { - const vec1 = new Float32Array([1, 0, 0]); - const vec2 = new Float32Array([-1, 0, 0]); - const similarity = TextEmbedder.cosineSimilarity(vec1, vec2); - - expect(similarity).toBeCloseTo(-1.0, 5); - }); - - it('should throw for vectors of different dimensions', () => { - const vec1 = new Float32Array([1, 2, 3]); - const vec2 = new Float32Array([1, 2]); - - expect(() => TextEmbedder.cosineSimilarity(vec1, vec2)).toThrow( - 'Embeddings must have the same dimension' - ); - }); - - it('should handle zero vectors', () => { - const vec1 = new Float32Array([0, 0, 0]); - const vec2 = new Float32Array([1, 2, 3]); - const similarity = TextEmbedder.cosineSimilarity(vec1, vec2); - - expect(similarity).toBe(0); - }); - }); - }); - - describe('VectorSearchManager', () => { - let tempDir: string; - let db: DatabaseConnection; - let searchManager: VectorSearchManager; - const TEST_DIMENSION = 3; // Use small dimension for tests - - beforeEach(() => { - tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-vector-test-')); - const dbPath = path.join(tempDir, 'test.db'); - db = DatabaseConnection.initialize(dbPath); - searchManager = createVectorSearch(db.getDb(), TEST_DIMENSION); - }); - - afterEach(() => { - db.close(); - if (fs.existsSync(tempDir)) { - fs.rmSync(tempDir, { recursive: true, force: true }); - } - }); - - it('should store and retrieve vectors', async () => { - await searchManager.initialize(); - - const embedding = new Float32Array([0.1, 0.2, 0.3]); - searchManager.storeVector('node1', embedding, 'test-model'); - - const retrieved = searchManager.getVector('node1'); - - expect(retrieved).not.toBeNull(); - expect(retrieved?.length).toBe(3); - expect(retrieved?.[0]).toBeCloseTo(0.1, 5); - }); - - it('should return null for non-existent vectors', async () => { - await searchManager.initialize(); - - const retrieved = searchManager.getVector('non-existent'); - - expect(retrieved).toBeNull(); - }); - - it('should check if vector exists', async () => { - await searchManager.initialize(); - - const embedding = new Float32Array([0.1, 0.2, 0.3]); - searchManager.storeVector('node1', embedding, 'test-model'); - - expect(searchManager.hasVector('node1')).toBe(true); - expect(searchManager.hasVector('node2')).toBe(false); - }); - - it('should delete vectors', async () => { - await searchManager.initialize(); - - const embedding = new Float32Array([0.1, 0.2, 0.3]); - searchManager.storeVector('node1', embedding, 'test-model'); - - expect(searchManager.hasVector('node1')).toBe(true); - - searchManager.deleteVector('node1'); - - expect(searchManager.hasVector('node1')).toBe(false); - }); - - it('should count vectors', async () => { - await searchManager.initialize(); - - expect(searchManager.getVectorCount()).toBe(0); - - searchManager.storeVector('node1', new Float32Array([0.1, 0.2, 0.3]), 'test'); - searchManager.storeVector('node2', new Float32Array([0.4, 0.5, 0.6]), 'test'); - - expect(searchManager.getVectorCount()).toBe(2); - }); - - it('should clear all vectors', async () => { - await searchManager.initialize(); - - searchManager.storeVector('node1', new Float32Array([0.1, 0.2, 0.3]), 'test'); - searchManager.storeVector('node2', new Float32Array([0.4, 0.5, 0.6]), 'test'); - - expect(searchManager.getVectorCount()).toBe(2); - - searchManager.clear(); - - expect(searchManager.getVectorCount()).toBe(0); - }); - - it('should perform brute-force similarity search', async () => { - await searchManager.initialize(); - - // Store some test vectors - searchManager.storeVector('node1', new Float32Array([1, 0, 0]), 'test'); - searchManager.storeVector('node2', new Float32Array([0.9, 0.1, 0]), 'test'); - searchManager.storeVector('node3', new Float32Array([0, 1, 0]), 'test'); - - // Search for similar to [1, 0, 0] - const query = new Float32Array([1, 0, 0]); - const results = searchManager.search(query, { limit: 3 }); - - expect(results.length).toBe(3); - expect(results[0].nodeId).toBe('node1'); // Most similar - expect(results[0].score).toBeCloseTo(1.0, 5); - expect(results[1].nodeId).toBe('node2'); // Second most similar - }); - - it('should respect minScore in search', async () => { - await searchManager.initialize(); - - searchManager.storeVector('node1', new Float32Array([1, 0, 0]), 'test'); - searchManager.storeVector('node2', new Float32Array([0, 1, 0]), 'test'); - - const query = new Float32Array([1, 0, 0]); - const results = searchManager.search(query, { limit: 10, minScore: 0.5 }); - - // Only node1 should match with score >= 0.5 - expect(results.length).toBe(1); - expect(results[0].nodeId).toBe('node1'); - }); - - it('should store vectors in batch', async () => { - await searchManager.initialize(); - - // Use normalized 3-dimensional vectors - const entries = [ - { nodeId: 'node1', embedding: new Float32Array([1.0, 0.0, 0.0]) }, - { nodeId: 'node2', embedding: new Float32Array([0.0, 1.0, 0.0]) }, - { nodeId: 'node3', embedding: new Float32Array([0.0, 0.0, 1.0]) }, - ]; - - searchManager.storeVectorBatch(entries, 'test-model'); - - expect(searchManager.getVectorCount()).toBe(3); - expect(searchManager.hasVector('node1')).toBe(true); - expect(searchManager.hasVector('node2')).toBe(true); - expect(searchManager.hasVector('node3')).toBe(true); - }); - - it('should get indexed node IDs', async () => { - await searchManager.initialize(); - - searchManager.storeVector('node1', new Float32Array([0.1, 0.2, 0.3]), 'test'); - searchManager.storeVector('node2', new Float32Array([0.4, 0.5, 0.6]), 'test'); - - const ids = searchManager.getIndexedNodeIds(); - - expect(ids).toContain('node1'); - expect(ids).toContain('node2'); - expect(ids.length).toBe(2); - }); - }); - - describe('CodeGraph Embedding Integration', () => { - let testDir: string; - let cg: CodeGraph; - - beforeEach(() => { - testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-embed-integration-')); - - // Create a simple test file - fs.writeFileSync( - path.join(testDir, 'test.ts'), - ` -export function processData(input: string): string { - return input.toUpperCase(); -} -` - ); - - cg = CodeGraph.initSync(testDir, { - config: { - include: ['**/*.ts'], - exclude: [], - }, - }); - }); - - afterEach(() => { - if (cg) { - cg.destroy(); - } - if (fs.existsSync(testDir)) { - fs.rmSync(testDir, { recursive: true, force: true }); - } - }); - - it('should report embeddings not initialized', () => { - expect(cg.isEmbeddingsInitialized()).toBe(false); - }); - - it('should return embedding stats even before initialization', () => { - const stats = cg.getEmbeddingStats(); - expect(stats).not.toBeNull(); - expect(stats!.totalVectors).toBe(0); - }); - - it('should throw when calling semanticSearch without initialization', async () => { - await expect(cg.semanticSearch('test')).rejects.toThrow(/not initialized/i); - }); - - it('should throw when calling findSimilar without initialization', async () => { - await expect(cg.findSimilar('test-id')).rejects.toThrow(/not initialized/i); - }); - }); -}); diff --git a/package.json b/package.json index f46a951..4d2769b 100644 --- a/package.json +++ b/package.json @@ -14,9 +14,8 @@ ], "scripts": { "build": "tsc && npm run copy-assets", - "postinstall": "node scripts/postinstall.js", "preuninstall": "node dist/bin/uninstall.js", - "copy-assets": "node -e \"const fs=require('fs');fs.mkdirSync('dist/db',{recursive:true});fs.copyFileSync('src/db/schema.sql','dist/db/schema.sql');fs.mkdirSync('dist/extraction/wasm',{recursive:true});fs.readdirSync('src/extraction/wasm').filter(f=>f.endsWith('.wasm')).forEach(f=>fs.copyFileSync('src/extraction/wasm/'+f,'dist/extraction/wasm/'+f));fs.mkdirSync('dist/visualizer/public',{recursive:true});fs.copyFileSync('src/visualizer/public/index.html','dist/visualizer/public/index.html')\"", + "copy-assets": "node -e \"const fs=require('fs');fs.mkdirSync('dist/db',{recursive:true});fs.copyFileSync('src/db/schema.sql','dist/db/schema.sql');fs.mkdirSync('dist/extraction/wasm',{recursive:true});fs.readdirSync('src/extraction/wasm').filter(f=>f.endsWith('.wasm')).forEach(f=>fs.copyFileSync('src/extraction/wasm/'+f,'dist/extraction/wasm/'+f))\"", "dev": "tsc --watch", "cli": "npm run build && node dist/bin/codegraph.js", "test": "vitest run", @@ -28,14 +27,12 @@ "keywords": [ "code-intelligence", "knowledge-graph", - "static-analysis", - "semantic-search" + "static-analysis" ], "author": "", "license": "MIT", "dependencies": { "@clack/prompts": "^1.2.0", - "@xenova/transformers": "^2.17.0", "commander": "^14.0.2", "node-sqlite3-wasm": "^0.8.30", "picomatch": "^4.0.3", @@ -50,8 +47,7 @@ "vitest": "^2.1.9" }, "optionalDependencies": { - "better-sqlite3": "^11.0.0", - "sqlite-vss": "^0.1.2" + "better-sqlite3": "^11.0.0" }, "engines": { "node": ">=18.0.0 <25.0.0" diff --git a/scripts/postinstall.js b/scripts/postinstall.js deleted file mode 100644 index 3725a5d..0000000 --- a/scripts/postinstall.js +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env node -/** - * Postinstall script - downloads the embedding model to ~/.codegraph/models - * This runs after `npm install` or `npx @colbymchenry/codegraph` - */ -const { existsSync, mkdirSync } = require('fs'); -const { join } = require('path'); -const { homedir } = require('os'); - -const CODEGRAPH_DIR = join(homedir(), '.codegraph'); -const MODELS_DIR = join(CODEGRAPH_DIR, 'models'); -const MODEL_ID = 'nomic-ai/nomic-embed-text-v1.5'; - -async function downloadModel() { - // Ensure directories exist - if (!existsSync(CODEGRAPH_DIR)) { - mkdirSync(CODEGRAPH_DIR, { recursive: true }); - } - if (!existsSync(MODELS_DIR)) { - mkdirSync(MODELS_DIR, { recursive: true }); - } - - // Check if model is already cached - const modelCachePath = join(MODELS_DIR, MODEL_ID.replace('/', '/')); - if (existsSync(modelCachePath)) { - console.log('Embedding model already downloaded.'); - return; - } - - console.log('Downloading embedding model (~130MB)...'); - console.log('This is a one-time download for semantic code search.\n'); - - try { - // Dynamic import for @xenova/transformers (ESM-only package) - const { pipeline, env } = await import('@xenova/transformers'); - - // Configure cache directory - env.cacheDir = MODELS_DIR; - - // Download with progress - await pipeline('feature-extraction', MODEL_ID, { - progress_callback: (progress) => { - if (progress.status === 'progress' && progress.file && progress.progress !== undefined) { - const fileName = progress.file.split('/').pop(); - const percent = Math.round(progress.progress); - process.stdout.write(`\rDownloading ${fileName}... ${percent}% `); - } else if (progress.status === 'done') { - process.stdout.write('\n'); - } - }, - }); - - console.log('\nEmbedding model ready!'); - } catch (error) { - // Don't fail the install if model download fails - // User can still use codegraph without semantic search - console.log('\nNote: Could not download embedding model.'); - console.log('Semantic search will download it on first use.'); - if (process.env.DEBUG) { - console.error(error); - } - } -} - -downloadModel().catch(() => { - // Silent exit - don't break npm install - process.exit(0); -}); diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index 1066a07..eb508b8 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -1078,69 +1078,6 @@ program } }); -/** - * codegraph visualize [path] - */ -program - .command('visualize [path]') - .description('Open interactive graph visualization in your browser') - .option('-p, --port ', 'Port to listen on (default: auto)', parseInt) - .option('--no-open', 'Do not open browser automatically') - .action(async (pathArg: string | undefined, options: { port?: number; open?: boolean }) => { - const projectPath = resolveProjectPath(pathArg); - - try { - if (!isInitialized(projectPath)) { - error(`CodeGraph not initialized in ${projectPath}`); - info('Run "codegraph init -i" first'); - process.exit(1); - } - - const { default: CodeGraph } = await loadCodeGraph(); - const cg = await CodeGraph.open(projectPath); - const stats = cg.getStats(); - - console.log(chalk.bold('\n CodeGraph Explorer\n')); - info(`Project: ${projectPath}`); - info(`Indexed: ${formatNumber(stats.nodeCount)} nodes, ${formatNumber(stats.edgeCount)} edges, ${formatNumber(stats.fileCount)} files\n`); - - const { VisualizerServer } = await import('../visualizer/server'); - const server = new VisualizerServer(cg); - const { url } = await server.start({ port: options.port, openBrowser: options.open !== false }); - - success(`Visualizer running at ${chalk.cyan(url)}`); - console.log(chalk.dim(' Press Ctrl+C to stop\n')); - - // Open browser - if (options.open !== false) { - const openCmd = process.platform === 'darwin' ? 'open' : - process.platform === 'win32' ? 'start' : 'xdg-open'; - spawn(openCmd, [url], { detached: true, stdio: 'ignore' }).unref(); - } - - // Handle shutdown — force exit on second Ctrl+C - let shuttingDown = false; - const shutdown = () => { - if (shuttingDown) { - process.exit(1); - } - shuttingDown = true; - console.log(chalk.dim('\n Shutting down...')); - server.stop().then(() => { - cg.close(); - process.exit(0); - }).catch(() => process.exit(1)); - // Force exit after 2s if graceful shutdown hangs - setTimeout(() => process.exit(1), 2000).unref(); - }; - process.on('SIGINT', shutdown); - process.on('SIGTERM', shutdown); - } catch (err) { - error(`Failed to start visualizer: ${err instanceof Error ? err.message : String(err)}`); - process.exit(1); - } - }); - /** * codegraph mark-dirty [path] * diff --git a/src/context/index.ts b/src/context/index.ts index 1d089b6..8ee2626 100644 --- a/src/context/index.ts +++ b/src/context/index.ts @@ -1,7 +1,7 @@ /** * Context Builder * - * Builds rich context for tasks by combining semantic search with graph traversal. + * Builds rich context for tasks by combining FTS search with graph traversal. * Outputs structured context ready to inject into Claude. */ @@ -22,7 +22,6 @@ import { } from '../types'; import { QueryBuilder } from '../db/queries'; import { GraphTraverser } from '../graph'; -import { VectorManager } from '../vectors'; import { formatContextAsMarkdown, formatContextAsJson } from './formatter'; import { logDebug } from '../errors'; import { validatePathWithinRoot } from '../utils'; @@ -185,18 +184,15 @@ export class ContextBuilder { private projectRoot: string; private queries: QueryBuilder; private traverser: GraphTraverser; - private vectorManager: VectorManager | null; constructor( projectRoot: string, queries: QueryBuilder, - traverser: GraphTraverser, - vectorManager: VectorManager | null + traverser: GraphTraverser ) { this.projectRoot = projectRoot; this.queries = queries; this.traverser = traverser; - this.vectorManager = vectorManager; } /** @@ -394,21 +390,7 @@ export class ContextBuilder { exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 3)); } - // Step 3: Try semantic search if vector manager is available - let semanticResults: SearchResult[] = []; - if (this.vectorManager && this.vectorManager.isInitialized()) { - try { - semanticResults = await this.vectorManager.search(query, { - limit: opts.searchLimit, - kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, - }); - logDebug('Semantic search results', { count: semanticResults.length }); - } catch (error) { - logDebug('Semantic search failed, falling back to text search', { query, error: String(error) }); - } - } - - // Step 4: Always run text search for natural language term matching + // Step 3: Run text search for natural language term matching // This catches file-name and node-name matches that semantic search may miss, // which is critical for template-heavy codebases (e.g., Liquid/Shopify themes) // where file names are the primary identifiers. @@ -457,7 +439,7 @@ export class ContextBuilder { logDebug('Text search failed', { query, error: String(error) }); } - // Step 5: Merge results, prioritizing exact matches, then text (path-boosted), then semantic + // Step 4: Merge results, prioritizing exact matches, then text (path-boosted) const seenIds = new Set(); let searchResults: SearchResult[] = []; @@ -477,14 +459,6 @@ export class ContextBuilder { } } - // Add semantic results - for (const result of semanticResults) { - if (!seenIds.has(result.node.id)) { - seenIds.add(result.node.id); - searchResults.push(result); - } - } - const queryLower = query.toLowerCase(); const isTestQuery = queryLower.includes('test') || queryLower.includes('spec'); @@ -1121,10 +1095,9 @@ export class ContextBuilder { export function createContextBuilder( projectRoot: string, queries: QueryBuilder, - traverser: GraphTraverser, - vectorManager: VectorManager | null + traverser: GraphTraverser ): ContextBuilder { - return new ContextBuilder(projectRoot, queries, traverser, vectorManager); + return new ContextBuilder(projectRoot, queries, traverser); } // Re-export formatter diff --git a/src/db/queries.ts b/src/db/queries.ts index 877f9e1..51f1a1a 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -1290,7 +1290,6 @@ export class QueryBuilder { this.nodeCache.clear(); this.db.transaction(() => { this.db.exec('DELETE FROM unresolved_refs'); - this.db.exec('DELETE FROM vectors'); this.db.exec('DELETE FROM edges'); this.db.exec('DELETE FROM nodes'); this.db.exec('DELETE FROM files'); diff --git a/src/db/schema.sql b/src/db/schema.sql index 39857e0..dd0a9f0 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -140,22 +140,6 @@ CREATE INDEX IF NOT EXISTS idx_unresolved_file_path ON unresolved_refs(file_path CREATE INDEX IF NOT EXISTS idx_unresolved_from_name ON unresolved_refs(from_node_id, reference_name); CREATE INDEX IF NOT EXISTS idx_edges_provenance ON edges(provenance); --- ============================================================================= --- Vector Storage (for future semantic search) --- ============================================================================= - --- Vector embeddings for semantic search --- Note: No foreign key constraint to allow standalone vector testing --- The VectorManager handles node-vector relationship at the application level -CREATE TABLE IF NOT EXISTS vectors ( - node_id TEXT PRIMARY KEY, - embedding BLOB NOT NULL, -- Float32 array stored as blob - model TEXT NOT NULL, -- Model used to generate embedding - created_at INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model); - -- Project metadata for version/provenance tracking CREATE TABLE IF NOT EXISTS project_metadata ( key TEXT PRIMARY KEY, diff --git a/src/index.ts b/src/index.ts index 778af86..6f37e87 100644 --- a/src/index.ts +++ b/src/index.ts @@ -46,7 +46,6 @@ import { ResolutionResult, } from './resolution'; import { GraphTraverser, GraphQueryManager } from './graph'; -import { VectorManager, createVectorManager, EmbeddingProgress } from './vectors'; import { ContextBuilder, createContextBuilder } from './context'; import { Mutex, FileLock } from './utils'; @@ -63,7 +62,6 @@ export { export { IndexProgress, IndexResult, SyncResult } from './extraction'; export { detectLanguage, isLanguageSupported, isGrammarLoaded, getSupportedLanguages, initGrammars, loadGrammarsForLanguages, loadAllGrammars } from './extraction'; export { ResolutionResult } from './resolution'; -export { EmbeddingProgress } from './vectors'; export { CodeGraphError, FileError, @@ -134,7 +132,6 @@ export class CodeGraph { private resolver: ReferenceResolver; private graphManager: GraphQueryManager; private traverser: GraphTraverser; - private vectorManager: VectorManager | null = null; private contextBuilder: ContextBuilder; // Mutex for preventing concurrent indexing operations (in-process) @@ -160,14 +157,10 @@ export class CodeGraph { this.resolver = createResolver(projectRoot, queries); this.graphManager = new GraphQueryManager(queries); this.traverser = new GraphTraverser(queries); - // Vector manager — always created, embeddings generated lazily on first use - this.vectorManager = createVectorManager(db.getDb(), queries, {}); - // Context builder (uses vector manager for semantic search) this.contextBuilder = createContextBuilder( projectRoot, queries, - this.traverser, - this.vectorManager + this.traverser ); } @@ -328,11 +321,6 @@ export class CodeGraph { close(): void { // Release file lock if held this.fileLock.release(); - // Dispose vector manager first to release ONNX workers - if (this.vectorManager) { - this.vectorManager.dispose(); - this.vectorManager = null; - } this.db.close(); } @@ -838,102 +826,6 @@ export class CodeGraph { return this.graphManager.getNodeMetrics(nodeId); } - // =========================================================================== - // Semantic Search (Vector Embeddings) - // =========================================================================== - - /** - * Initialize the embedding system - * - * This downloads the embedding model on first use and initializes - * the vector search system. Must be called before using semantic search. - */ - async initializeEmbeddings(): Promise { - if (!this.vectorManager) { - this.vectorManager = createVectorManager(this.db.getDb(), this.queries, { - embedder: { - showProgress: true, - }, - }); - } - await this.vectorManager.initialize(); - } - - /** - * Check if embeddings are initialized - */ - isEmbeddingsInitialized(): boolean { - return this.vectorManager?.isInitialized() ?? false; - } - - /** - * Generate embeddings for all eligible nodes - * - * @param onProgress - Optional progress callback - * @returns Number of nodes embedded - */ - async generateEmbeddings( - onProgress?: (progress: EmbeddingProgress) => void - ): Promise { - if (!this.vectorManager) { - await this.initializeEmbeddings(); - } - return this.vectorManager!.embedAllNodes(onProgress); - } - - /** - * Semantic search using embeddings - * - * Searches for code nodes semantically similar to the query. - * Requires embeddings to be initialized first. - * - * @param query - Natural language search query - * @param limit - Maximum number of results (default: 10) - * @returns Array of search results with similarity scores - */ - async semanticSearch(query: string, limit: number = 10): Promise { - if (!this.vectorManager || !this.vectorManager.isInitialized()) { - throw new Error( - 'Embeddings not initialized. Call initializeEmbeddings() first.' - ); - } - return this.vectorManager.search(query, { limit }); - } - - /** - * Find similar code blocks - * - * Finds nodes semantically similar to a given node. - * Requires embeddings to be initialized first. - * - * @param nodeId - ID of the node to find similar nodes for - * @param limit - Maximum number of results (default: 10) - * @returns Array of similar nodes with similarity scores - */ - async findSimilar(nodeId: string, limit: number = 10): Promise { - if (!this.vectorManager || !this.vectorManager.isInitialized()) { - throw new Error( - 'Embeddings not initialized. Call initializeEmbeddings() first.' - ); - } - return this.vectorManager.findSimilar(nodeId, { limit }); - } - - /** - * Get vector embedding statistics - */ - getEmbeddingStats(): { - totalVectors: number; - vssEnabled: boolean; - modelId: string; - dimension: number; - } | null { - if (!this.vectorManager) { - return null; - } - return this.vectorManager.getStats(); - } - // =========================================================================== // Context Building // =========================================================================== @@ -964,13 +856,6 @@ export class CodeGraph { query: string, options?: FindRelevantContextOptions ): Promise { - // Update context builder with current vector manager - this.contextBuilder = createContextBuilder( - this.projectRoot, - this.queries, - this.traverser, - this.vectorManager - ); return this.contextBuilder.findRelevantContext(query, options); } @@ -978,7 +863,7 @@ export class CodeGraph { * Build context for a task * * Creates comprehensive context by: - * 1. Running semantic search to find entry points + * 1. Running FTS search to find entry points * 2. Expanding the graph around entry points * 3. Extracting code blocks for key nodes * 4. Formatting output for Claude @@ -991,13 +876,6 @@ export class CodeGraph { input: TaskInput, options?: BuildContextOptions ): Promise { - // Update context builder with current vector manager - this.contextBuilder = createContextBuilder( - this.projectRoot, - this.queries, - this.traverser, - this.vectorManager - ); return this.contextBuilder.buildContext(input, options); } diff --git a/src/vectors/embedder.ts b/src/vectors/embedder.ts deleted file mode 100644 index d72321a..0000000 --- a/src/vectors/embedder.ts +++ /dev/null @@ -1,410 +0,0 @@ -/** - * Text Embedder - * - * Generates vector embeddings using the nomic-embed-text model via Transformers.js. - * Uses ONNX runtime under the hood for fast local inference. - */ - -import * as path from 'path'; -import * as fs from 'fs'; -import { homedir } from 'os'; - -// Global model cache directory - uses codegraph's models directory for shared embedding models -const GLOBAL_MODELS_DIR = path.join(homedir(), '.codegraph', 'models'); - -// Dynamic import for @xenova/transformers (ESM-only package) -// We use dynamic import to support CommonJS builds -let transformersModule: typeof import('@xenova/transformers') | null = null; - -async function getTransformers() { - if (!transformersModule) { - transformersModule = await import('@xenova/transformers'); - } - return transformersModule; -} - -// Type for the feature extraction pipeline -type FeatureExtractionPipeline = any; - -/** - * Default model for embeddings - * nomic-embed-text-v1.5 produces 384-dimensional embeddings - */ -export const DEFAULT_MODEL = 'nomic-ai/nomic-embed-text-v1.5'; -export const EMBEDDING_DIMENSION = 768; // nomic-embed-text-v1.5 uses 768 dimensions - -/** - * Options for the embedder - */ -export interface EmbedderOptions { - /** Model ID to use (default: nomic-ai/nomic-embed-text-v1.5) */ - modelId?: string; - - /** Directory to cache the model (default: ~/.codegraph/models) */ - cacheDir?: string; - - /** Whether to show progress during model download */ - showProgress?: boolean; -} - -/** - * Text embedding result - */ -export interface EmbeddingResult { - /** The embedding vector */ - embedding: Float32Array; - - /** Dimension of the embedding */ - dimension: number; - - /** Model used to generate the embedding */ - model: string; -} - -/** - * Batch embedding result - */ -export interface BatchEmbeddingResult { - /** Array of embeddings in same order as input */ - embeddings: Float32Array[]; - - /** Dimension of each embedding */ - dimension: number; - - /** Model used to generate embeddings */ - model: string; - - /** Processing time in milliseconds */ - durationMs: number; -} - -/** - * Text Embedder using Transformers.js - * - * Uses the nomic-embed-text-v1.5 model to generate embeddings for code - * and natural language queries. - */ -export class TextEmbedder { - private modelId: string; - private cacheDir: string; - private pipeline: FeatureExtractionPipeline | null = null; - private initialized = false; - private showProgress: boolean; - - constructor(options: EmbedderOptions = {}) { - this.modelId = options.modelId || DEFAULT_MODEL; - this.cacheDir = options.cacheDir || GLOBAL_MODELS_DIR; - this.showProgress = options.showProgress ?? false; - } - - /** - * Initialize the embedder by loading the model - * - * This will download the model on first use if not already cached. - */ - async initialize(): Promise { - if (this.initialized) { - return; - } - - // Load transformers.js dynamically (ESM-only package) - const { pipeline, env } = await getTransformers(); - - // Configure transformers.js to use local cache - env.cacheDir = this.cacheDir; - - // Ensure cache directory exists - if (!fs.existsSync(this.cacheDir)) { - fs.mkdirSync(this.cacheDir, { recursive: true }); - } - - // Disable remote model checking if model is already cached - // This speeds up initialization significantly - const modelCacheExists = fs.existsSync( - path.join(this.cacheDir, this.modelId.replace('/', '--')) - ); - if (modelCacheExists) { - env.allowRemoteModels = false; - } - - // Load the pipeline with quantized model to reduce WASM memory pressure. - // Quantized (int8/uint8) is ~4x smaller than FP32 with minimal quality loss. - this.pipeline = await pipeline('feature-extraction', this.modelId, { - quantized: true, - progress_callback: this.showProgress - ? (progress: { status: string; file?: string; progress?: number }) => { - if (progress.status === 'progress' && progress.file && progress.progress) { - const pct = Math.round(progress.progress); - process.stdout.write(`\rDownloading ${progress.file}: ${pct}%\x1b[K`); - } else if (progress.status === 'done') { - process.stdout.write('\n'); - } - } - : undefined, - }); - - this.initialized = true; - } - - /** - * Check if the embedder is initialized - */ - isInitialized(): boolean { - return this.initialized; - } - - /** - * Get the model ID being used - */ - getModelId(): string { - return this.modelId; - } - - /** - * Get the embedding dimension - */ - getDimension(): number { - return EMBEDDING_DIMENSION; - } - - /** - * Generate embedding for a single text - * - * @param text - Text to embed - * @returns Embedding result - */ - async embed(text: string): Promise { - if (!this.initialized || !this.pipeline) { - throw new Error('Embedder not initialized. Call initialize() first.'); - } - - // Prepare text for nomic-embed-text (it expects specific prefixes) - const preparedText = this.prepareText(text, 'document'); - - // Generate embedding - const output = await this.pipeline(preparedText, { - pooling: 'mean', - normalize: true, - }); - - // Extract the embedding array - handle various data formats - const data = output.data as unknown; - const embedding = this.toFloat32Array(data); - - return { - embedding, - dimension: embedding.length, - model: this.modelId, - }; - } - - /** - * Generate embedding for a query (uses different prefix) - * - * @param query - Query text to embed - * @returns Embedding result - */ - async embedQuery(query: string): Promise { - if (!this.initialized || !this.pipeline) { - throw new Error('Embedder not initialized. Call initialize() first.'); - } - - // Prepare text for nomic-embed-text query - const preparedText = this.prepareText(query, 'search_query'); - - // Generate embedding - const output = await this.pipeline(preparedText, { - pooling: 'mean', - normalize: true, - }); - - // Extract the embedding array - handle various data formats - const data = output.data as unknown; - const embedding = this.toFloat32Array(data); - - return { - embedding, - dimension: embedding.length, - model: this.modelId, - }; - } - - /** - * Generate embeddings for multiple texts in a batch - * - * @param texts - Array of texts to embed - * @param type - Type of text (document or search_query) - * @returns Batch embedding result - */ - async embedBatch( - texts: string[], - type: 'document' | 'search_query' = 'document' - ): Promise { - if (!this.initialized || !this.pipeline) { - throw new Error('Embedder not initialized. Call initialize() first.'); - } - - if (texts.length === 0) { - return { - embeddings: [], - dimension: EMBEDDING_DIMENSION, - model: this.modelId, - durationMs: 0, - }; - } - - const startTime = Date.now(); - - // Prepare all texts - const preparedTexts = texts.map((t) => this.prepareText(t, type)); - - // Generate embeddings - const outputs = await this.pipeline(preparedTexts, { - pooling: 'mean', - normalize: true, - }); - - // Extract embeddings - const embeddings: Float32Array[] = []; - const dims = outputs.dims as number[]; - const dimension = dims[1] ?? EMBEDDING_DIMENSION; - const data = outputs.data as unknown; - const flatData = this.toFloat32Array(data); - - for (let i = 0; i < texts.length; i++) { - const start = i * dimension; - const end = start + dimension; - embeddings.push(flatData.slice(start, end)); - } - - return { - embeddings, - dimension, - model: this.modelId, - durationMs: Date.now() - startTime, - }; - } - - /** - * Convert various array formats to Float32Array - */ - private toFloat32Array(data: unknown): Float32Array { - if (data instanceof Float32Array) { - return data; - } - if (Array.isArray(data)) { - return new Float32Array(data); - } - if (data && typeof data === 'object' && 'length' in data) { - // Handle TypedArray-like objects - const arr = data as ArrayLike; - return Float32Array.from(Array.from(arr)); - } - throw new Error('Unsupported data format for embedding'); - } - - /** - * Prepare text for the nomic-embed-text model - * - * The model expects specific prefixes for different tasks: - * - "search_document: " for documents to be searched - * - "search_query: " for search queries - */ - private prepareText(text: string, type: 'document' | 'search_query'): string { - // Truncate very long texts (model has a max token limit) - const maxLength = 8192; // nomic-embed-text-v1.5 supports 8192 tokens - const truncatedText = text.length > maxLength ? text.slice(0, maxLength) : text; - - // Add appropriate prefix - if (type === 'search_query') { - return `search_query: ${truncatedText}`; - } else { - return `search_document: ${truncatedText}`; - } - } - - /** - * Create text representation of a code node for embedding - * - * Combines name, signature, docstring, and code snippet into - * a searchable text representation. - */ - static createNodeText(node: { - name: string; - kind: string; - qualifiedName?: string; - signature?: string; - docstring?: string; - filePath: string; - }): string { - const parts: string[] = []; - - // Add kind and name - parts.push(`${node.kind}: ${node.name}`); - - // Add qualified name if different from name - if (node.qualifiedName && node.qualifiedName !== node.name) { - parts.push(`path: ${node.qualifiedName}`); - } - - // Add file path - parts.push(`file: ${node.filePath}`); - - // Add signature if present - if (node.signature) { - parts.push(`signature: ${node.signature}`); - } - - // Add docstring if present - if (node.docstring) { - parts.push(`documentation: ${node.docstring}`); - } - - return parts.join('\n'); - } - - /** - * Compute cosine similarity between two embeddings - */ - static cosineSimilarity(a: Float32Array, b: Float32Array): number { - if (a.length !== b.length) { - throw new Error('Embeddings must have the same dimension'); - } - - let dotProduct = 0; - let normA = 0; - let normB = 0; - - for (let i = 0; i < a.length; i++) { - const aVal = a[i]!; - const bVal = b[i]!; - dotProduct += aVal * bVal; - normA += aVal * aVal; - normB += bVal * bVal; - } - - normA = Math.sqrt(normA); - normB = Math.sqrt(normB); - - if (normA === 0 || normB === 0) { - return 0; - } - - return dotProduct / (normA * normB); - } - - /** - * Release resources - */ - dispose(): void { - this.pipeline = null; - this.initialized = false; - } -} - -/** - * Create a text embedder instance - */ -export function createEmbedder(options?: EmbedderOptions): TextEmbedder { - return new TextEmbedder(options); -} diff --git a/src/vectors/index.ts b/src/vectors/index.ts deleted file mode 100644 index 06cbbe9..0000000 --- a/src/vectors/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Vectors Module - * - * Provides text embedding and vector similarity search for semantic code search. - */ - -export { - TextEmbedder, - createEmbedder, - DEFAULT_MODEL, - EMBEDDING_DIMENSION, - EmbedderOptions, - EmbeddingResult, - BatchEmbeddingResult, -} from './embedder'; - -export { - VectorSearchManager, - createVectorSearch, - VectorSearchOptions, -} from './search'; - -export { - VectorManager, - createVectorManager, - VectorManagerOptions, - EmbeddingProgress, -} from './manager'; diff --git a/src/vectors/manager.ts b/src/vectors/manager.ts deleted file mode 100644 index 3f8ac3d..0000000 --- a/src/vectors/manager.ts +++ /dev/null @@ -1,363 +0,0 @@ -/** - * Vector Manager - * - * High-level manager that coordinates embedding generation and vector search. - */ - -import { SqliteDatabase } from '../db/sqlite-adapter'; -import { Node, SearchResult, SearchOptions } from '../types'; -import { TextEmbedder, createEmbedder, EmbedderOptions, EMBEDDING_DIMENSION } from './embedder'; -import { VectorSearchManager, createVectorSearch } from './search'; -import { QueryBuilder } from '../db/queries'; - -/** - * Progress callback for embedding generation - */ -export interface EmbeddingProgress { - /** Current node index */ - current: number; - - /** Total nodes to embed */ - total: number; - - /** Current node being embedded */ - nodeName?: string; -} - -/** - * Options for the vector manager - */ -export interface VectorManagerOptions { - /** Embedder options */ - embedder?: EmbedderOptions; - - /** Node kinds to embed (default: functions, methods, classes, interfaces) */ - nodeKinds?: Node['kind'][]; - - /** Batch size for embedding generation */ - batchSize?: number; -} - -/** - * Default node kinds to embed - */ -const DEFAULT_NODE_KINDS: Node['kind'][] = [ - 'function', - 'method', - 'class', - 'interface', - 'type_alias', - 'module', - 'component', -]; - -/** - * Vector Manager - * - * Provides high-level interface for semantic search: - * - Generates embeddings for code nodes - * - Stores embeddings in the database - * - Performs semantic similarity search - */ -export class VectorManager { - private embedder: TextEmbedder; - private searchManager: VectorSearchManager; - private queries: QueryBuilder; - private nodeKinds: Node['kind'][]; - private batchSize: number; - private initialized = false; - - constructor( - db: SqliteDatabase, - queries: QueryBuilder, - options: VectorManagerOptions = {} - ) { - this.embedder = createEmbedder(options.embedder); - this.searchManager = createVectorSearch(db, EMBEDDING_DIMENSION); - this.queries = queries; - this.nodeKinds = options.nodeKinds || DEFAULT_NODE_KINDS; - this.batchSize = options.batchSize || 32; - } - - /** - * Initialize the vector manager - * - * Loads the embedding model and initializes vector search. - */ - async initialize(): Promise { - if (this.initialized) { - return; - } - - // Initialize embedder (downloads model if needed) - await this.embedder.initialize(); - - // Initialize vector search (loads sqlite-vss if available) - await this.searchManager.initialize(); - - this.initialized = true; - } - - /** - * Check if the vector manager is initialized - */ - isInitialized(): boolean { - return this.initialized; - } - - /** - * Generate embeddings for all eligible nodes - * - * @param onProgress - Optional progress callback - * @returns Number of nodes embedded - */ - async embedAllNodes(onProgress?: (progress: EmbeddingProgress) => void): Promise { - if (!this.initialized) { - throw new Error('VectorManager not initialized. Call initialize() first.'); - } - - // Get all nodes that should be embedded - const nodesToEmbed: Node[] = []; - for (const kind of this.nodeKinds) { - const nodes = this.queries.getNodesByKind(kind); - nodesToEmbed.push(...nodes); - } - - // Filter out nodes that already have embeddings - const existingIds = new Set(this.searchManager.getIndexedNodeIds()); - const newNodes = nodesToEmbed.filter((n) => !existingIds.has(n.id)); - - if (newNodes.length === 0) { - return 0; - } - - // Process in batches - let processed = 0; - const model = this.embedder.getModelId(); - - for (let i = 0; i < newNodes.length; i += this.batchSize) { - const batch = newNodes.slice(i, i + this.batchSize); - - // Create text representations - const texts = batch.map((node) => TextEmbedder.createNodeText(node)); - - // Generate embeddings - const result = await this.embedder.embedBatch(texts, 'document'); - - // Store embeddings - const entries: Array<{ nodeId: string; embedding: Float32Array }> = []; - for (let idx = 0; idx < batch.length; idx++) { - const node = batch[idx]; - const embedding = result.embeddings[idx]; - if (node && embedding) { - entries.push({ nodeId: node.id, embedding }); - } - } - this.searchManager.storeVectorBatch(entries, model); - - processed += batch.length; - - // Report progress - if (onProgress) { - onProgress({ - current: processed, - total: newNodes.length, - nodeName: batch[batch.length - 1]?.name, - }); - } - } - - return processed; - } - - /** - * Generate embedding for a single node - * - * @param node - Node to embed - */ - async embedNode(node: Node): Promise { - if (!this.initialized) { - throw new Error('VectorManager not initialized. Call initialize() first.'); - } - - const text = TextEmbedder.createNodeText(node); - const result = await this.embedder.embed(text); - this.searchManager.storeVector(node.id, result.embedding, result.model); - } - - /** - * Semantic search for nodes matching a query - * - * @param query - Natural language query - * @param options - Search options - * @returns Array of search results with similarity scores - */ - async search(query: string, options: SearchOptions = {}): Promise { - if (!this.initialized) { - throw new Error('VectorManager not initialized. Call initialize() first.'); - } - - const { limit = 10, kinds } = options; - - // Generate query embedding - const queryResult = await this.embedder.embedQuery(query); - - // Search for similar vectors - const vectorResults = this.searchManager.search(queryResult.embedding, { - limit: limit * 2, // Get more results to filter - minScore: 0.3, // Minimum similarity threshold - }); - - // Get nodes and filter by kind if specified - const results: SearchResult[] = []; - for (const vr of vectorResults) { - const node = this.queries.getNodeById(vr.nodeId); - if (!node) { - continue; - } - - // Filter by node kind if specified - if (kinds && kinds.length > 0 && !kinds.includes(node.kind)) { - continue; - } - - results.push({ - node, - score: vr.score, - }); - - if (results.length >= limit) { - break; - } - } - - return results; - } - - /** - * Find nodes similar to a given node - * - * @param nodeId - ID of the node to find similar nodes for - * @param options - Search options - * @returns Array of similar nodes with similarity scores - */ - async findSimilar(nodeId: string, options: SearchOptions = {}): Promise { - if (!this.initialized) { - throw new Error('VectorManager not initialized. Call initialize() first.'); - } - - const { limit = 10, kinds } = options; - - // Get the node's embedding - let embedding = this.searchManager.getVector(nodeId); - - // If no embedding exists, generate one - if (!embedding) { - const node = this.queries.getNodeById(nodeId); - if (!node) { - throw new Error(`Node not found: ${nodeId}`); - } - - await this.embedNode(node); - embedding = this.searchManager.getVector(nodeId); - - if (!embedding) { - throw new Error(`Failed to generate embedding for node: ${nodeId}`); - } - } - - // Search for similar vectors (excluding the source node) - const vectorResults = this.searchManager.search(embedding, { - limit: limit + 1, // Get one extra to exclude the source - minScore: 0.3, - }); - - // Get nodes and filter - const results: SearchResult[] = []; - for (const vr of vectorResults) { - // Skip the source node - if (vr.nodeId === nodeId) { - continue; - } - - const node = this.queries.getNodeById(vr.nodeId); - if (!node) { - continue; - } - - // Filter by node kind if specified - if (kinds && kinds.length > 0 && !kinds.includes(node.kind)) { - continue; - } - - results.push({ - node, - score: vr.score, - }); - - if (results.length >= limit) { - break; - } - } - - return results; - } - - /** - * Delete embedding for a node - * - * @param nodeId - ID of the node - */ - deleteNodeEmbedding(nodeId: string): void { - this.searchManager.deleteVector(nodeId); - } - - /** - * Get statistics about vector storage - */ - getStats(): { - totalVectors: number; - vssEnabled: boolean; - modelId: string; - dimension: number; - } { - return { - totalVectors: this.searchManager.getVectorCount(), - vssEnabled: this.searchManager.isVssEnabled(), - modelId: this.embedder.getModelId(), - dimension: this.embedder.getDimension(), - }; - } - - /** - * Clear all vectors - */ - clear(): void { - this.searchManager.clear(); - } - - /** - * Rebuild the VSS index - */ - rebuildIndex(): void { - this.searchManager.rebuildVssIndex(); - } - - /** - * Release resources - */ - dispose(): void { - this.embedder.dispose(); - } -} - -/** - * Create a vector manager - */ -export function createVectorManager( - db: SqliteDatabase, - queries: QueryBuilder, - options?: VectorManagerOptions -): VectorManager { - return new VectorManager(db, queries, options); -} diff --git a/src/vectors/search.ts b/src/vectors/search.ts deleted file mode 100644 index bdde94d..0000000 --- a/src/vectors/search.ts +++ /dev/null @@ -1,472 +0,0 @@ -/** - * Vector Search - * - * Provides vector similarity search using sqlite-vss extension. - * Falls back to brute-force cosine similarity if sqlite-vss is not available. - */ - -import { SqliteDatabase } from '../db/sqlite-adapter'; -import { Node } from '../types'; -import { TextEmbedder, EMBEDDING_DIMENSION } from './embedder'; - -/** - * Options for vector search - */ -export interface VectorSearchOptions { - /** Maximum number of results to return */ - limit?: number; - - /** Minimum similarity score (0-1) */ - minScore?: number; - - /** Node kinds to filter results */ - nodeKinds?: Node['kind'][]; -} - -/** - * Vector Search Manager - * - * Handles vector storage and similarity search for semantic code search. - */ -export class VectorSearchManager { - private db: SqliteDatabase; - private vssEnabled = false; - private embeddingDimension: number; - - constructor(db: SqliteDatabase, dimension: number = EMBEDDING_DIMENSION) { - this.db = db; - this.embeddingDimension = dimension; - } - - /** - * Initialize vector search - * - * Attempts to load sqlite-vss extension. Falls back to brute-force - * search if the extension is not available. - */ - async initialize(): Promise { - try { - // Try to load sqlite-vss extension - await this.loadVssExtension(); - this.vssEnabled = true; - console.log('sqlite-vss extension loaded successfully'); - - // Create the VSS virtual table - this.createVssTable(); - } catch (error) { - // Fall back to brute-force search - console.warn( - 'sqlite-vss extension not available, falling back to brute-force search:', - error instanceof Error ? error.message : String(error) - ); - this.vssEnabled = false; - } - - // Ensure the vectors table exists (for both VSS and fallback modes) - this.ensureVectorsTable(); - } - - /** - * Load the sqlite-vss extension - */ - private async loadVssExtension(): Promise { - try { - // The sqlite-vss npm package provides functions to load extensions - const vss = await import('sqlite-vss'); - - // Use the load function which loads both vector0 and vss0 - // VSS extension expects the raw better-sqlite3 Database instance - if (typeof vss.load === 'function') { - vss.load(this.db as any); - } else if (typeof vss.default?.load === 'function') { - vss.default.load(this.db as any); - } else { - throw new Error('sqlite-vss load function not found'); - } - } catch (error) { - throw new Error(`Failed to load sqlite-vss: ${error instanceof Error ? error.message : String(error)}`); - } - } - - /** - * Create the VSS virtual table for vector search - */ - private createVssTable(): void { - // Check if the table already exists - const tableExists = this.db - .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='vss_vectors'") - .get(); - - if (!tableExists) { - // Create VSS virtual table - // vss0 is the vector search extension - this.db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS vss_vectors USING vss0( - embedding(${this.embeddingDimension}) - ); - `); - - // Create mapping table to link VSS rowids to node IDs - this.db.exec(` - CREATE TABLE IF NOT EXISTS vss_map ( - rowid INTEGER PRIMARY KEY, - node_id TEXT NOT NULL UNIQUE - ); - `); - - // Create index on node_id - this.db.exec(` - CREATE INDEX IF NOT EXISTS idx_vss_map_node ON vss_map(node_id); - `); - } - } - - /** - * Ensure the basic vectors table exists (for fallback mode) - */ - private ensureVectorsTable(): void { - this.db.exec(` - CREATE TABLE IF NOT EXISTS vectors ( - node_id TEXT PRIMARY KEY, - embedding BLOB NOT NULL, - model TEXT NOT NULL, - created_at INTEGER NOT NULL - ); - `); - } - - /** - * Check if VSS extension is enabled - */ - isVssEnabled(): boolean { - return this.vssEnabled; - } - - /** - * Store a vector embedding for a node - * - * @param nodeId - ID of the node - * @param embedding - Vector embedding - * @param model - Model used to generate embedding - */ - storeVector(nodeId: string, embedding: Float32Array, model: string): void { - const now = Date.now(); - - // Store in the vectors table (always, for persistence) - const blob = Buffer.from(embedding.buffer); - this.db - .prepare( - ` - INSERT OR REPLACE INTO vectors (node_id, embedding, model, created_at) - VALUES (?, ?, ?, ?) - ` - ) - .run(nodeId, blob, model, now); - - // Also store in VSS table if enabled - if (this.vssEnabled) { - this.storeInVss(nodeId, embedding); - } - } - - /** - * Store vector in VSS virtual table - */ - private storeInVss(nodeId: string, embedding: Float32Array): void { - try { - // Check if already exists - const existing = this.db - .prepare('SELECT rowid FROM vss_map WHERE node_id = ?') - .get(nodeId) as { rowid: number } | undefined; - - if (existing) { - // Update existing vector - const vectorJson = JSON.stringify(Array.from(embedding)); - this.db - .prepare('UPDATE vss_vectors SET embedding = ? WHERE rowid = ?') - .run(vectorJson, existing.rowid); - } else { - // Insert new vector - get max rowid and increment - const maxRow = this.db - .prepare('SELECT MAX(rowid) as max FROM vss_map') - .get() as { max: number | null } | undefined; - const newRowid = (maxRow?.max ?? 0) + 1; - - const vectorJson = JSON.stringify(Array.from(embedding)); - this.db - .prepare('INSERT INTO vss_vectors (rowid, embedding) VALUES (?, ?)') - .run(newRowid, vectorJson); - - // Map the rowid to node_id - this.db - .prepare('INSERT INTO vss_map (rowid, node_id) VALUES (?, ?)') - .run(newRowid, nodeId); - } - } catch (error) { - // VSS operations can fail for various reasons (dimension mismatch, etc.) - // Fall back to brute-force search silently - console.warn( - 'VSS storage failed, using brute-force search:', - error instanceof Error ? error.message : String(error) - ); - } - } - - /** - * Store multiple vectors in a batch - * - * @param entries - Array of node IDs and embeddings - * @param model - Model used to generate embeddings - */ - storeVectorBatch( - entries: Array<{ nodeId: string; embedding: Float32Array }>, - model: string - ): void { - const now = Date.now(); - - // Use a transaction for better performance - this.db.transaction(() => { - for (const entry of entries) { - const blob = Buffer.from(entry.embedding.buffer); - this.db - .prepare( - ` - INSERT OR REPLACE INTO vectors (node_id, embedding, model, created_at) - VALUES (?, ?, ?, ?) - ` - ) - .run(entry.nodeId, blob, model, now); - - if (this.vssEnabled) { - this.storeInVss(entry.nodeId, entry.embedding); - } - } - })(); - } - - /** - * Get vector for a node - * - * @param nodeId - ID of the node - * @returns Embedding or null if not found - */ - getVector(nodeId: string): Float32Array | null { - const row = this.db - .prepare('SELECT embedding FROM vectors WHERE node_id = ?') - .get(nodeId) as { embedding: Buffer } | undefined; - - if (!row) { - return null; - } - - return new Float32Array(row.embedding.buffer.slice( - row.embedding.byteOffset, - row.embedding.byteOffset + row.embedding.byteLength - )); - } - - /** - * Delete vector for a node - * - * @param nodeId - ID of the node - */ - deleteVector(nodeId: string): void { - this.db.prepare('DELETE FROM vectors WHERE node_id = ?').run(nodeId); - - if (this.vssEnabled) { - // Get the rowid before deleting - const mapping = this.db - .prepare('SELECT rowid FROM vss_map WHERE node_id = ?') - .get(nodeId) as { rowid: number } | undefined; - - if (mapping) { - this.db.prepare('DELETE FROM vss_vectors WHERE rowid = ?').run(mapping.rowid); - this.db.prepare('DELETE FROM vss_map WHERE node_id = ?').run(nodeId); - } - } - } - - /** - * Search for similar vectors - * - * @param queryEmbedding - Query vector to search for - * @param options - Search options - * @returns Array of node IDs with similarity scores - */ - search( - queryEmbedding: Float32Array, - options: VectorSearchOptions = {} - ): Array<{ nodeId: string; score: number }> { - const { limit = 10, minScore = 0 } = options; - - if (this.vssEnabled) { - return this.searchWithVss(queryEmbedding, limit, minScore); - } else { - return this.searchBruteForce(queryEmbedding, limit, minScore); - } - } - - /** - * Search using sqlite-vss KNN search - */ - private searchWithVss( - queryEmbedding: Float32Array, - limit: number, - minScore: number - ): Array<{ nodeId: string; score: number }> { - try { - const vectorJson = JSON.stringify(Array.from(queryEmbedding)); - // Sanitize limit to prevent SQL injection (ensure it's a positive integer) - const safeLimit = Math.max(1, Math.floor(limit)); - - // Use VSS KNN search - // The distance is L2 (euclidean), we need to convert to similarity score - // Note: sqlite-vss requires LIMIT to be a literal, not a parameter - const rows = this.db - .prepare( - ` - SELECT m.node_id, v.distance - FROM ( - SELECT rowid, distance - FROM vss_vectors - WHERE vss_search(embedding, ?) - LIMIT ${safeLimit} - ) v - JOIN vss_map m ON m.rowid = v.rowid - ` - ) - .all(vectorJson) as Array<{ node_id: string; distance: number }>; - - // Convert L2 distance to similarity score (1 / (1 + distance)) - return rows - .map((row) => ({ - nodeId: row.node_id, - score: 1 / (1 + row.distance), - })) - .filter((r) => r.score >= minScore); - } catch (error) { - // VSS search failed, fall back to brute force - console.warn( - 'VSS search failed, using brute-force:', - error instanceof Error ? error.message : String(error) - ); - return this.searchBruteForce(queryEmbedding, limit, minScore); - } - } - - /** - * Brute-force search using cosine similarity - */ - private searchBruteForce( - queryEmbedding: Float32Array, - limit: number, - minScore: number - ): Array<{ nodeId: string; score: number }> { - // Get all vectors - const rows = this.db - .prepare('SELECT node_id, embedding FROM vectors') - .all() as Array<{ node_id: string; embedding: Buffer }>; - - // Calculate cosine similarity for each - const results: Array<{ nodeId: string; score: number }> = []; - - for (const row of rows) { - const embedding = new Float32Array(row.embedding.buffer.slice( - row.embedding.byteOffset, - row.embedding.byteOffset + row.embedding.byteLength - )); - - const score = TextEmbedder.cosineSimilarity(queryEmbedding, embedding); - - if (score >= minScore) { - results.push({ nodeId: row.node_id, score }); - } - } - - // Sort by score descending and limit - results.sort((a, b) => b.score - a.score); - return results.slice(0, limit); - } - - /** - * Get count of stored vectors - */ - getVectorCount(): number { - const result = this.db - .prepare('SELECT COUNT(*) as count FROM vectors') - .get() as { count: number }; - return result.count; - } - - /** - * Check if a node has a vector - */ - hasVector(nodeId: string): boolean { - const result = this.db - .prepare('SELECT 1 FROM vectors WHERE node_id = ? LIMIT 1') - .get(nodeId); - return !!result; - } - - /** - * Get all node IDs that have vectors - */ - getIndexedNodeIds(): string[] { - const rows = this.db - .prepare('SELECT node_id FROM vectors') - .all() as Array<{ node_id: string }>; - return rows.map((r) => r.node_id); - } - - /** - * Clear all vectors - */ - clear(): void { - this.db.prepare('DELETE FROM vectors').run(); - - if (this.vssEnabled) { - this.db.prepare('DELETE FROM vss_vectors').run(); - this.db.prepare('DELETE FROM vss_map').run(); - } - } - - /** - * Rebuild VSS index from vectors table - * - * Useful after bulk operations or if VSS index gets out of sync. - */ - rebuildVssIndex(): void { - if (!this.vssEnabled) { - return; - } - - // Clear VSS tables - this.db.prepare('DELETE FROM vss_vectors').run(); - this.db.prepare('DELETE FROM vss_map').run(); - - // Reload from vectors table - const rows = this.db - .prepare('SELECT node_id, embedding FROM vectors') - .all() as Array<{ node_id: string; embedding: Buffer }>; - - this.db.transaction(() => { - for (const row of rows) { - const embedding = new Float32Array(row.embedding.buffer.slice( - row.embedding.byteOffset, - row.embedding.byteOffset + row.embedding.byteLength - )); - this.storeInVss(row.node_id, embedding); - } - })(); - } -} - -/** - * Create a vector search manager - */ -export function createVectorSearch( - db: SqliteDatabase, - dimension?: number -): VectorSearchManager { - return new VectorSearchManager(db, dimension); -} diff --git a/src/visualizer/public/index.html b/src/visualizer/public/index.html deleted file mode 100644 index 741e518..0000000 --- a/src/visualizer/public/index.html +++ /dev/null @@ -1,1994 +0,0 @@ - - - - - - CodeGraph Explorer - - - - - - - - - - - - - -
- - - - -
- - - - -
-
-
-
🔮
-
Search for a starting point
-
Type a symbol name, pick it, and trace its call chain
-
- -
- - - -
-
- - -
-
-
-
-
-
- -
-
-
-
-
- - -
-
-
🧠
-
Enable Semantic Search
-
- CodeGraph Explorer uses semantic embeddings to understand your code by meaning, not just keywords. - This lets you ask questions like "how does authentication work?" and get accurate results. -

- This is a one-time setup that generates a local embedding model for this project. No data leaves your machine. -
-
-
-
-
-
- Preparing... - 0% -
-
-
- - -
-
-
- - -
-
→ Expand Callees
-
← Expand Callers
-
-
🌐 Full Call Graph
-
💥 Impact Analysis
-
-
📂 Show Children
-
📋 View Details
-
-
✖ Remove from Graph
-
- - -
- - -
- - - - diff --git a/src/visualizer/server.ts b/src/visualizer/server.ts deleted file mode 100644 index 58f8688..0000000 --- a/src/visualizer/server.ts +++ /dev/null @@ -1,521 +0,0 @@ -/** - * CodeGraph Visualizer Server - * - * Lightweight HTTP server that serves the graph visualization UI - * and exposes REST API endpoints for querying the CodeGraph database. - */ - -import * as http from 'http'; -import * as fs from 'fs'; -import * as path from 'path'; -import * as url from 'url'; -import { execFile } from 'child_process'; -import type CodeGraph from '../index'; -import type { Node, Edge, NodeKind } from '../types'; - -export interface VisualizerOptions { - /** Port to listen on (0 = auto-assign) */ - port?: number; - /** Whether to open browser automatically */ - openBrowser?: boolean; - /** Host to bind to */ - host?: string; -} - -/** - * Serialize a Subgraph (which uses Map) to plain JSON - */ -function serializeSubgraph(subgraph: { nodes: Map; edges: Edge[]; roots: string[] }) { - return { - nodes: Array.from(subgraph.nodes.values()), - edges: subgraph.edges, - roots: subgraph.roots, - }; -} - -export class VisualizerServer { - private cg: CodeGraph; - private server: http.Server | null = null; - private projectRoot: string; - private symbolIndexCache: string | null = null; - private claudeAvailable: boolean | null = null; - - constructor(cg: CodeGraph) { - this.cg = cg; - this.projectRoot = cg.getProjectRoot(); - } - - /** - * Build a compact symbol index string for Claude prompts - */ - private buildSymbolIndex(): string { - if (this.symbolIndexCache) return this.symbolIndexCache; - - const validKinds: NodeKind[] = ['function', 'method', 'class', 'interface', 'component', 'route', 'enum', 'type_alias']; - const byFile = new Map(); - - for (const kind of validKinds) { - for (const node of this.cg.getNodesByKind(kind)) { - const symbols = byFile.get(node.filePath) || []; - symbols.push(`${node.kind}:${node.name}`); - byFile.set(node.filePath, symbols); - } - } - - const lines: string[] = []; - for (const [file, symbols] of byFile) { - lines.push(`${file}: ${symbols.join(', ')}`); - } - - this.symbolIndexCache = lines.join('\n'); - return this.symbolIndexCache; - } - - /** - * Ask Claude CLI to interpret a natural language question into relevant symbol names - */ - private async askClaude(question: string): Promise { - // Check if claude is available (cache result) - if (this.claudeAvailable === false) return null; - - const symbolIndex = this.buildSymbolIndex(); - - const prompt = `Given the question and codebase symbol index below, identify the single best ENTRY POINT symbol — the one function, component, or route handler where this flow starts. - -Rules: -- Pick ONE symbol that is the starting point a user or request would hit first -- Prefer page components, route handlers, or top-level functions -- Do NOT pick utility functions, helpers, or middleware - -Return ONLY this JSON, nothing else: -{"entry": "symbolName"} - -Question: "${question}" - -Symbol index: -${symbolIndex}`; - - return new Promise((resolve) => { - const timeout = setTimeout(() => { - resolve(null); - }, 30000); - - execFile('claude', ['-p', prompt, '--output-format', 'text'], { - timeout: 30000, - maxBuffer: 1024 * 1024, - }, (err, stdout) => { - clearTimeout(timeout); - - if (err) { - this.claudeAvailable = false; - resolve(null); - return; - } - - this.claudeAvailable = true; - - // Parse Claude's response — try object format first, then array fallback - try { - const text = stdout.trim(); - // Try to extract JSON object {"entry": ..., "flow": [...]} - const objMatch = text.match(/\{[\s\S]*\}/); - if (objMatch) { - const parsed = JSON.parse(objMatch[0]) as { entry?: string; flow?: string[] }; - if (parsed.flow && Array.isArray(parsed.flow) && parsed.flow.length > 0) { - // Return flow with entry first - const names = parsed.flow.map(String); - if (parsed.entry && !names.includes(parsed.entry)) { - names.unshift(String(parsed.entry)); - } - resolve(names); - return; - } - } - // Fallback: try JSON array - const arrMatch = text.match(/\[[\s\S]*\]/); - if (arrMatch) { - const names = JSON.parse(arrMatch[0]) as string[]; - if (Array.isArray(names) && names.length > 0) { - resolve(names.map(String)); - return; - } - } - } catch { - // Parse failed - } - - resolve(null); - }); - }); - } - - /** - * Start the visualizer server - */ - async start(options: VisualizerOptions = {}): Promise<{ port: number; url: string }> { - const host = options.host || '127.0.0.1'; - const port = options.port || 0; - - this.server = http.createServer((req, res) => { - this.handleRequest(req, res).catch((err) => { - console.error('[Visualizer] Request error:', err); - res.writeHead(500, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'Internal server error' })); - }); - }); - - return new Promise((resolve, reject) => { - this.server!.listen(port, host, () => { - const addr = this.server!.address(); - if (!addr || typeof addr === 'string') { - reject(new Error('Failed to get server address')); - return; - } - const serverUrl = `http://${host}:${addr.port}`; - resolve({ port: addr.port, url: serverUrl }); - }); - - this.server!.on('error', reject); - }); - } - - /** - * Stop the server - */ - stop(): Promise { - return new Promise((resolve) => { - if (this.server) { - this.server.close(() => resolve()); - } else { - resolve(); - } - }); - } - - private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise { - const parsedUrl = url.parse(req.url || '/', true); - const pathname = parsedUrl.pathname || '/'; - - // CORS headers for local development - res.setHeader('Access-Control-Allow-Origin', '*'); - res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS'); - res.setHeader('Access-Control-Allow-Headers', 'Content-Type'); - - if (req.method === 'OPTIONS') { - res.writeHead(204); - res.end(); - return; - } - - // API routes - if (pathname.startsWith('/api/')) { - return this.handleAPI(pathname, parsedUrl.query as Record, res); - } - - // Static file serving - return this.serveStatic(pathname, res); - } - - private async handleAPI( - pathname: string, - query: Record, - res: http.ServerResponse - ): Promise { - const json = (data: unknown, status = 200) => { - res.writeHead(status, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(data)); - }; - - try { - // GET /api/status - if (pathname === '/api/status') { - const stats = this.cg.getStats(); - json({ stats, projectRoot: this.projectRoot, projectName: path.basename(this.projectRoot) }); - return; - } - - // GET /api/embeddings/status - if (pathname === '/api/embeddings/status') { - const embeddingStats = this.cg.getEmbeddingStats(); - const isInitialized = this.cg.isEmbeddingsInitialized(); - const totalVectors = embeddingStats?.totalVectors ?? 0; - const stats = this.cg.getStats(); - // Consider ready if we have vectors for at least half the eligible nodes - const eligibleNodes = stats.nodeCount - (stats.nodesByKind.file ?? 0) - (stats.nodesByKind.import ?? 0); - const isReady = totalVectors > 0 && totalVectors >= eligibleNodes * 0.5; - json({ isEnabled: true, isInitialized, isReady, totalVectors, eligibleNodes }); - return; - } - - // GET /api/embeddings/generate — SSE stream that enables, initializes, and generates embeddings - if (pathname === '/api/embeddings/generate') { - res.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive', - }); - - const send = (event: string, data: unknown) => { - res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); - }; - - try { - // Step 1: Initialize embedding model (downloads on first use) - send('status', { phase: 'model', message: 'Loading embedding model (first time may download ~30MB)...' }); - await this.cg.initializeEmbeddings(); - send('status', { phase: 'model', message: 'Embedding model ready' }); - - // Step 3: Generate embeddings with progress - send('status', { phase: 'embedding', message: 'Generating embeddings...' }); - const count = await this.cg.generateEmbeddings((progress) => { - send('progress', { - current: progress.current, - total: progress.total, - nodeName: progress.nodeName, - percent: progress.total > 0 ? Math.round((progress.current / progress.total) * 100) : 0, - }); - }); - - send('complete', { totalEmbedded: count, message: `Generated ${count} embeddings` }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - send('error', { message }); - } - - res.end(); - return; - } - - // GET /api/search?q=...&kind=...&limit=... - if (pathname === '/api/search') { - const q = query.q || ''; - const kind = query.kind as NodeKind | undefined; - const limit = parseInt(query.limit || '30', 10); - if (!q) { - json({ results: [] }); - return; - } - const results = this.cg.searchNodes(q, { kinds: kind ? [kind] : undefined, limit }); - json({ results }); - return; - } - - // GET /api/explore?q=... - // Find the best entry point, then return its call graph - if (pathname === '/api/explore') { - const q = query.q || ''; - if (!q) { - json({ nodes: [], edges: [], roots: [], entryPoint: null }); - return; - } - - let entryNodeId: string | null = null; - let usedClaude = false; - const validKinds: NodeKind[] = ['function', 'method', 'class', 'interface', 'component', 'route']; - - // Try Claude CLI to find the best entry point - const claudeNames = await this.askClaude(q); - if (claudeNames && claudeNames.length > 0) { - usedClaude = true; - // Find the entry point in the graph - for (const name of claudeNames) { - if (entryNodeId) break; - const results = this.cg.searchNodes(name, { kinds: validKinds, limit: 3 }); - for (const r of results) { - if (r.node.name.toLowerCase() === name.toLowerCase() || - r.node.name.toLowerCase().includes(name.toLowerCase()) || - name.toLowerCase().includes(r.node.name.toLowerCase())) { - entryNodeId = r.node.id; - break; - } - } - } - } - - // Keyword fallback: find best match from query keywords - if (!entryNodeId) { - const stopWords = new Set(['how', 'does', 'what', 'the', 'is', 'a', 'an', 'and', 'or', 'in', 'to', 'for', 'of', 'with', 'when', 'do', 'it', 'my', 'work', 'works', 'about', 'show', 'me']); - const keywords = q.toLowerCase().split(/\s+/) - .map(w => w.replace(/[^a-z0-9]/g, '')) - .filter(w => w.length >= 2 && !stopWords.has(w)); - - for (const kw of keywords) { - if (entryNodeId) break; - const results = this.cg.searchNodes(kw, { kinds: validKinds, limit: 5 }); - if (results.length > 0) { - entryNodeId = results[0]!.node.id; - } - } - } - - if (!entryNodeId) { - json({ nodes: [], edges: [], roots: [], entryPoint: null }); - return; - } - - // Get the call graph from this entry point (depth 3) - const callGraph = this.cg.getCallGraph(entryNodeId, 3); - const result = serializeSubgraph(callGraph); - - json({ - nodes: result.nodes, - edges: result.edges, - roots: [entryNodeId], - entryPoint: entryNodeId, - usedClaude, - }); - return; - } - - // GET /api/overview?limit=... - if (pathname === '/api/overview') { - const limit = parseInt(query.limit || '50', 10); - // Get top-level exported classes, functions, components - const kinds: NodeKind[] = ['class', 'function', 'interface', 'component', 'enum', 'type_alias']; - const nodes: Node[] = []; - for (const kind of kinds) { - const kindNodes = this.cg.getNodesByKind(kind); - for (const n of kindNodes) { - if (n.isExported || n.kind === 'class' || n.kind === 'component') { - nodes.push(n); - } - if (nodes.length >= limit) break; - } - if (nodes.length >= limit) break; - } - json({ nodes }); - return; - } - - // GET /api/files - if (pathname === '/api/files') { - const files = this.cg.getFiles(); - json({ files }); - return; - } - - // Routes with node ID: /api/node//... - const nodeMatch = pathname.match(/^\/api\/node\/([^/]+)(\/.*)?$/); - if (nodeMatch) { - const nodeId = decodeURIComponent(nodeMatch[1]!); - const sub = nodeMatch[2] || ''; - - // GET /api/node/ - if (!sub || sub === '/') { - const node = this.cg.getNode(nodeId); - if (!node) { - json({ error: 'Node not found' }, 404); - return; - } - const code = await this.cg.getCode(nodeId); - const ancestors = this.cg.getAncestors(nodeId); - json({ node, code, ancestors }); - return; - } - - // GET /api/node//callers?depth=... - if (sub === '/callers') { - const depth = parseInt(query.depth || '1', 10); - const items = this.cg.getCallers(nodeId, depth); - json({ items }); - return; - } - - // GET /api/node//callees?depth=... - if (sub === '/callees') { - const depth = parseInt(query.depth || '1', 10); - const items = this.cg.getCallees(nodeId, depth); - json({ items }); - return; - } - - // GET /api/node//children - if (sub === '/children') { - const children = this.cg.getChildren(nodeId); - json({ children }); - return; - } - - // GET /api/node//impact?depth=... - if (sub === '/impact') { - const depth = parseInt(query.depth || '2', 10); - const subgraph = this.cg.getImpactRadius(nodeId, depth); - json(serializeSubgraph(subgraph)); - return; - } - - // GET /api/node//callgraph?depth=... - if (sub === '/callgraph') { - const depth = parseInt(query.depth || '2', 10); - const subgraph = this.cg.getCallGraph(nodeId, depth); - json(serializeSubgraph(subgraph)); - return; - } - - // GET /api/node//context - if (sub === '/context') { - const context = this.cg.getContext(nodeId); - json({ context }); - return; - } - - json({ error: 'Unknown endpoint' }, 404); - return; - } - - // GET /api/file-nodes?path=... - if (pathname === '/api/file-nodes') { - const filePath = query.path || ''; - if (!filePath) { - json({ error: 'path parameter required' }, 400); - return; - } - const nodes = this.cg.getNodesInFile(filePath); - json({ nodes }); - return; - } - - json({ error: 'Unknown API endpoint' }, 404); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - json({ error: message }, 500); - } - } - - private serveStatic(pathname: string, res: http.ServerResponse): void { - if (pathname === '/' || pathname === '/index.html') { - pathname = '/index.html'; - } - - // Resolve from the public directory next to this file - const publicDir = path.join(__dirname, 'public'); - const filePath = path.join(publicDir, pathname); - - // Security: prevent directory traversal - if (!filePath.startsWith(publicDir)) { - res.writeHead(403); - res.end('Forbidden'); - return; - } - - const ext = path.extname(filePath).toLowerCase(); - const mimeTypes: Record = { - '.html': 'text/html', - '.css': 'text/css', - '.js': 'application/javascript', - '.json': 'application/json', - '.png': 'image/png', - '.svg': 'image/svg+xml', - '.ico': 'image/x-icon', - }; - - try { - const content = fs.readFileSync(filePath); - res.writeHead(200, { 'Content-Type': mimeTypes[ext] || 'application/octet-stream' }); - res.end(content); - } catch { - res.writeHead(404, { 'Content-Type': 'text/plain' }); - res.end('Not found'); - } - } -}