refactor: Remove semantic search and vector embedding functionality
Removes @xenova/transformers dependency, vector storage tables, embedding generation, and semantic search APIs. Simplifies context building to use only FTS search. Eliminates visualizer server, postinstall model download, and related CLI commands. Reduces package size and complexity while maintaining core static analysis capabilities.
This commit is contained in:
Colby McHenry
@@ -58,10 +58,6 @@ src/
|
||||
│ ├── index.ts # GraphQueryManager
|
||||
│ ├── traversal.ts # GraphTraverser (BFS/DFS, impact radius)
|
||||
│ └── queries.ts # High-level graph queries
|
||||
├── vectors/ # Semantic search with embeddings
|
||||
│ ├── index.ts # VectorManager
|
||||
│ ├── embedder.ts # ONNX runtime + model loading
|
||||
│ └── search.ts # Similarity search
|
||||
├── context/ # Context building for AI assistants
|
||||
│ ├── index.ts # ContextBuilder
|
||||
│ └── formatter.ts # Markdown/JSON output formatting
|
||||
@@ -83,14 +79,12 @@ src/
|
||||
|
||||
### Key Classes
|
||||
|
||||
- **CodeGraph** (`src/index.ts`): Main entry point. Lifecycle methods (`init`, `open`, `close`), indexing (`indexAll`, `sync`), graph queries (`traverse`, `getCallGraph`, `getImpactRadius`), semantic search (`semanticSearch`, `findSimilar`), context building (`buildContext`)
|
||||
- **CodeGraph** (`src/index.ts`): Main entry point. Lifecycle methods (`init`, `open`, `close`), indexing (`indexAll`, `sync`), graph queries (`traverse`, `getCallGraph`, `getImpactRadius`), context building (`buildContext`)
|
||||
|
||||
- **ExtractionOrchestrator** (`src/extraction/index.ts`): Coordinates file scanning, parsing, and storing. Uses tree-sitter native bindings for each supported language
|
||||
|
||||
- **GraphTraverser** (`src/graph/traversal.ts`): BFS/DFS traversal, call graph construction, impact radius calculation, path finding
|
||||
|
||||
- **VectorManager** (`src/vectors/manager.ts`): Manages embeddings using `@xenova/transformers` for ONNX inference. Stores vectors in SQLite BLOB format
|
||||
|
||||
- **ReferenceResolver** (`src/resolution/index.ts`): Resolves unresolved references after full indexing using framework patterns, import resolution, and name matching
|
||||
|
||||
### Database Schema
|
||||
@@ -100,7 +94,6 @@ SQLite database with:
|
||||
- `edges`: Relationships (calls, imports, extends, contains, etc.)
|
||||
- `files`: Tracked source files with content hashes
|
||||
- `unresolved_refs`: References pending resolution
|
||||
- `vectors`: Embeddings stored as BLOBs
|
||||
- `nodes_fts`: FTS5 virtual table for full-text search
|
||||
|
||||
### Supported Languages
|
||||
@@ -153,7 +146,6 @@ Tests are in `__tests__/` directory with files mirroring the module structure:
|
||||
- `extraction.test.ts` - Tree-sitter parsing for all languages
|
||||
- `resolution.test.ts` - Reference resolution
|
||||
- `graph.test.ts` - Traversal and graph queries
|
||||
- `vectors.test.ts` - Embedding and semantic search
|
||||
- `context.test.ts` - Context building
|
||||
- `sync.test.ts` - Incremental updates and git hooks
|
||||
|
||||
|
||||
@@ -275,18 +275,6 @@ describe('CodeGraph Foundation', () => {
|
||||
cg.close();
|
||||
});
|
||||
|
||||
it('should require embedding initialization for semantic search', async () => {
|
||||
const cg = CodeGraph.initSync(tempDir);
|
||||
|
||||
// Semantic search requires embeddings to be initialized first
|
||||
await expect(cg.semanticSearch('test')).rejects.toThrow(/not initialized/i);
|
||||
await expect(cg.findSimilar('test')).rejects.toThrow(/not initialized/i);
|
||||
|
||||
// Check embedding status
|
||||
expect(cg.isEmbeddingsInitialized()).toBe(false);
|
||||
|
||||
cg.close();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,303 +0,0 @@
|
||||
/**
|
||||
* Vector Embedding Tests
|
||||
*
|
||||
* Tests for vector embedding and semantic search functionality.
|
||||
* Note: Full embedding tests require the model to be downloaded,
|
||||
* which can take time on first run.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import CodeGraph from '../src/index';
|
||||
import { TextEmbedder } from '../src/vectors/embedder';
|
||||
import { VectorSearchManager, createVectorSearch } from '../src/vectors/search';
|
||||
import { DatabaseConnection } from '../src/db';
|
||||
|
||||
describe('Vector Embeddings', () => {
|
||||
describe('TextEmbedder', () => {
|
||||
describe('createNodeText', () => {
|
||||
it('should create text representation from node', () => {
|
||||
const node = {
|
||||
name: 'processPayment',
|
||||
kind: 'function',
|
||||
qualifiedName: 'PaymentService.processPayment',
|
||||
signature: '(amount: number) => Promise<Receipt>',
|
||||
docstring: 'Process a payment and return a receipt.',
|
||||
filePath: 'src/services/payment.ts',
|
||||
};
|
||||
|
||||
const text = TextEmbedder.createNodeText(node);
|
||||
|
||||
expect(text).toContain('function: processPayment');
|
||||
expect(text).toContain('path: PaymentService.processPayment');
|
||||
expect(text).toContain('file: src/services/payment.ts');
|
||||
expect(text).toContain('signature: (amount: number) => Promise<Receipt>');
|
||||
expect(text).toContain('documentation: Process a payment');
|
||||
});
|
||||
|
||||
it('should handle minimal node data', () => {
|
||||
const node = {
|
||||
name: 'helper',
|
||||
kind: 'function',
|
||||
filePath: 'src/utils.ts',
|
||||
};
|
||||
|
||||
const text = TextEmbedder.createNodeText(node);
|
||||
|
||||
expect(text).toContain('function: helper');
|
||||
expect(text).toContain('file: src/utils.ts');
|
||||
expect(text).not.toContain('signature:');
|
||||
expect(text).not.toContain('documentation:');
|
||||
});
|
||||
});
|
||||
|
||||
describe('cosineSimilarity', () => {
|
||||
it('should compute similarity between identical vectors', () => {
|
||||
const vec = new Float32Array([0.1, 0.2, 0.3, 0.4, 0.5]);
|
||||
const similarity = TextEmbedder.cosineSimilarity(vec, vec);
|
||||
|
||||
expect(similarity).toBeCloseTo(1.0, 5);
|
||||
});
|
||||
|
||||
it('should compute similarity between orthogonal vectors', () => {
|
||||
const vec1 = new Float32Array([1, 0, 0]);
|
||||
const vec2 = new Float32Array([0, 1, 0]);
|
||||
const similarity = TextEmbedder.cosineSimilarity(vec1, vec2);
|
||||
|
||||
expect(similarity).toBeCloseTo(0.0, 5);
|
||||
});
|
||||
|
||||
it('should compute similarity between opposite vectors', () => {
|
||||
const vec1 = new Float32Array([1, 0, 0]);
|
||||
const vec2 = new Float32Array([-1, 0, 0]);
|
||||
const similarity = TextEmbedder.cosineSimilarity(vec1, vec2);
|
||||
|
||||
expect(similarity).toBeCloseTo(-1.0, 5);
|
||||
});
|
||||
|
||||
it('should throw for vectors of different dimensions', () => {
|
||||
const vec1 = new Float32Array([1, 2, 3]);
|
||||
const vec2 = new Float32Array([1, 2]);
|
||||
|
||||
expect(() => TextEmbedder.cosineSimilarity(vec1, vec2)).toThrow(
|
||||
'Embeddings must have the same dimension'
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle zero vectors', () => {
|
||||
const vec1 = new Float32Array([0, 0, 0]);
|
||||
const vec2 = new Float32Array([1, 2, 3]);
|
||||
const similarity = TextEmbedder.cosineSimilarity(vec1, vec2);
|
||||
|
||||
expect(similarity).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('VectorSearchManager', () => {
|
||||
let tempDir: string;
|
||||
let db: DatabaseConnection;
|
||||
let searchManager: VectorSearchManager;
|
||||
const TEST_DIMENSION = 3; // Use small dimension for tests
|
||||
|
||||
beforeEach(() => {
|
||||
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-vector-test-'));
|
||||
const dbPath = path.join(tempDir, 'test.db');
|
||||
db = DatabaseConnection.initialize(dbPath);
|
||||
searchManager = createVectorSearch(db.getDb(), TEST_DIMENSION);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
if (fs.existsSync(tempDir)) {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('should store and retrieve vectors', async () => {
|
||||
await searchManager.initialize();
|
||||
|
||||
const embedding = new Float32Array([0.1, 0.2, 0.3]);
|
||||
searchManager.storeVector('node1', embedding, 'test-model');
|
||||
|
||||
const retrieved = searchManager.getVector('node1');
|
||||
|
||||
expect(retrieved).not.toBeNull();
|
||||
expect(retrieved?.length).toBe(3);
|
||||
expect(retrieved?.[0]).toBeCloseTo(0.1, 5);
|
||||
});
|
||||
|
||||
it('should return null for non-existent vectors', async () => {
|
||||
await searchManager.initialize();
|
||||
|
||||
const retrieved = searchManager.getVector('non-existent');
|
||||
|
||||
expect(retrieved).toBeNull();
|
||||
});
|
||||
|
||||
it('should check if vector exists', async () => {
|
||||
await searchManager.initialize();
|
||||
|
||||
const embedding = new Float32Array([0.1, 0.2, 0.3]);
|
||||
searchManager.storeVector('node1', embedding, 'test-model');
|
||||
|
||||
expect(searchManager.hasVector('node1')).toBe(true);
|
||||
expect(searchManager.hasVector('node2')).toBe(false);
|
||||
});
|
||||
|
||||
it('should delete vectors', async () => {
|
||||
await searchManager.initialize();
|
||||
|
||||
const embedding = new Float32Array([0.1, 0.2, 0.3]);
|
||||
searchManager.storeVector('node1', embedding, 'test-model');
|
||||
|
||||
expect(searchManager.hasVector('node1')).toBe(true);
|
||||
|
||||
searchManager.deleteVector('node1');
|
||||
|
||||
expect(searchManager.hasVector('node1')).toBe(false);
|
||||
});
|
||||
|
||||
it('should count vectors', async () => {
|
||||
await searchManager.initialize();
|
||||
|
||||
expect(searchManager.getVectorCount()).toBe(0);
|
||||
|
||||
searchManager.storeVector('node1', new Float32Array([0.1, 0.2, 0.3]), 'test');
|
||||
searchManager.storeVector('node2', new Float32Array([0.4, 0.5, 0.6]), 'test');
|
||||
|
||||
expect(searchManager.getVectorCount()).toBe(2);
|
||||
});
|
||||
|
||||
it('should clear all vectors', async () => {
|
||||
await searchManager.initialize();
|
||||
|
||||
searchManager.storeVector('node1', new Float32Array([0.1, 0.2, 0.3]), 'test');
|
||||
searchManager.storeVector('node2', new Float32Array([0.4, 0.5, 0.6]), 'test');
|
||||
|
||||
expect(searchManager.getVectorCount()).toBe(2);
|
||||
|
||||
searchManager.clear();
|
||||
|
||||
expect(searchManager.getVectorCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('should perform brute-force similarity search', async () => {
|
||||
await searchManager.initialize();
|
||||
|
||||
// Store some test vectors
|
||||
searchManager.storeVector('node1', new Float32Array([1, 0, 0]), 'test');
|
||||
searchManager.storeVector('node2', new Float32Array([0.9, 0.1, 0]), 'test');
|
||||
searchManager.storeVector('node3', new Float32Array([0, 1, 0]), 'test');
|
||||
|
||||
// Search for similar to [1, 0, 0]
|
||||
const query = new Float32Array([1, 0, 0]);
|
||||
const results = searchManager.search(query, { limit: 3 });
|
||||
|
||||
expect(results.length).toBe(3);
|
||||
expect(results[0].nodeId).toBe('node1'); // Most similar
|
||||
expect(results[0].score).toBeCloseTo(1.0, 5);
|
||||
expect(results[1].nodeId).toBe('node2'); // Second most similar
|
||||
});
|
||||
|
||||
it('should respect minScore in search', async () => {
|
||||
await searchManager.initialize();
|
||||
|
||||
searchManager.storeVector('node1', new Float32Array([1, 0, 0]), 'test');
|
||||
searchManager.storeVector('node2', new Float32Array([0, 1, 0]), 'test');
|
||||
|
||||
const query = new Float32Array([1, 0, 0]);
|
||||
const results = searchManager.search(query, { limit: 10, minScore: 0.5 });
|
||||
|
||||
// Only node1 should match with score >= 0.5
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0].nodeId).toBe('node1');
|
||||
});
|
||||
|
||||
it('should store vectors in batch', async () => {
|
||||
await searchManager.initialize();
|
||||
|
||||
// Use normalized 3-dimensional vectors
|
||||
const entries = [
|
||||
{ nodeId: 'node1', embedding: new Float32Array([1.0, 0.0, 0.0]) },
|
||||
{ nodeId: 'node2', embedding: new Float32Array([0.0, 1.0, 0.0]) },
|
||||
{ nodeId: 'node3', embedding: new Float32Array([0.0, 0.0, 1.0]) },
|
||||
];
|
||||
|
||||
searchManager.storeVectorBatch(entries, 'test-model');
|
||||
|
||||
expect(searchManager.getVectorCount()).toBe(3);
|
||||
expect(searchManager.hasVector('node1')).toBe(true);
|
||||
expect(searchManager.hasVector('node2')).toBe(true);
|
||||
expect(searchManager.hasVector('node3')).toBe(true);
|
||||
});
|
||||
|
||||
it('should get indexed node IDs', async () => {
|
||||
await searchManager.initialize();
|
||||
|
||||
searchManager.storeVector('node1', new Float32Array([0.1, 0.2, 0.3]), 'test');
|
||||
searchManager.storeVector('node2', new Float32Array([0.4, 0.5, 0.6]), 'test');
|
||||
|
||||
const ids = searchManager.getIndexedNodeIds();
|
||||
|
||||
expect(ids).toContain('node1');
|
||||
expect(ids).toContain('node2');
|
||||
expect(ids.length).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('CodeGraph Embedding Integration', () => {
|
||||
let testDir: string;
|
||||
let cg: CodeGraph;
|
||||
|
||||
beforeEach(() => {
|
||||
testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-embed-integration-'));
|
||||
|
||||
// Create a simple test file
|
||||
fs.writeFileSync(
|
||||
path.join(testDir, 'test.ts'),
|
||||
`
|
||||
export function processData(input: string): string {
|
||||
return input.toUpperCase();
|
||||
}
|
||||
`
|
||||
);
|
||||
|
||||
cg = CodeGraph.initSync(testDir, {
|
||||
config: {
|
||||
include: ['**/*.ts'],
|
||||
exclude: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (cg) {
|
||||
cg.destroy();
|
||||
}
|
||||
if (fs.existsSync(testDir)) {
|
||||
fs.rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('should report embeddings not initialized', () => {
|
||||
expect(cg.isEmbeddingsInitialized()).toBe(false);
|
||||
});
|
||||
|
||||
it('should return embedding stats even before initialization', () => {
|
||||
const stats = cg.getEmbeddingStats();
|
||||
expect(stats).not.toBeNull();
|
||||
expect(stats!.totalVectors).toBe(0);
|
||||
});
|
||||
|
||||
it('should throw when calling semanticSearch without initialization', async () => {
|
||||
await expect(cg.semanticSearch('test')).rejects.toThrow(/not initialized/i);
|
||||
});
|
||||
|
||||
it('should throw when calling findSimilar without initialization', async () => {
|
||||
await expect(cg.findSimilar('test-id')).rejects.toThrow(/not initialized/i);
|
||||
});
|
||||
});
|
||||
});
|
||||
+3
-7
@@ -14,9 +14,8 @@
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsc && npm run copy-assets",
|
||||
"postinstall": "node scripts/postinstall.js",
|
||||
"preuninstall": "node dist/bin/uninstall.js",
|
||||
"copy-assets": "node -e \"const fs=require('fs');fs.mkdirSync('dist/db',{recursive:true});fs.copyFileSync('src/db/schema.sql','dist/db/schema.sql');fs.mkdirSync('dist/extraction/wasm',{recursive:true});fs.readdirSync('src/extraction/wasm').filter(f=>f.endsWith('.wasm')).forEach(f=>fs.copyFileSync('src/extraction/wasm/'+f,'dist/extraction/wasm/'+f));fs.mkdirSync('dist/visualizer/public',{recursive:true});fs.copyFileSync('src/visualizer/public/index.html','dist/visualizer/public/index.html')\"",
|
||||
"copy-assets": "node -e \"const fs=require('fs');fs.mkdirSync('dist/db',{recursive:true});fs.copyFileSync('src/db/schema.sql','dist/db/schema.sql');fs.mkdirSync('dist/extraction/wasm',{recursive:true});fs.readdirSync('src/extraction/wasm').filter(f=>f.endsWith('.wasm')).forEach(f=>fs.copyFileSync('src/extraction/wasm/'+f,'dist/extraction/wasm/'+f))\"",
|
||||
"dev": "tsc --watch",
|
||||
"cli": "npm run build && node dist/bin/codegraph.js",
|
||||
"test": "vitest run",
|
||||
@@ -28,14 +27,12 @@
|
||||
"keywords": [
|
||||
"code-intelligence",
|
||||
"knowledge-graph",
|
||||
"static-analysis",
|
||||
"semantic-search"
|
||||
"static-analysis"
|
||||
],
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@clack/prompts": "^1.2.0",
|
||||
"@xenova/transformers": "^2.17.0",
|
||||
"commander": "^14.0.2",
|
||||
"node-sqlite3-wasm": "^0.8.30",
|
||||
"picomatch": "^4.0.3",
|
||||
@@ -50,8 +47,7 @@
|
||||
"vitest": "^2.1.9"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"better-sqlite3": "^11.0.0",
|
||||
"sqlite-vss": "^0.1.2"
|
||||
"better-sqlite3": "^11.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0 <25.0.0"
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Postinstall script - downloads the embedding model to ~/.codegraph/models
|
||||
* This runs after `npm install` or `npx @colbymchenry/codegraph`
|
||||
*/
|
||||
const { existsSync, mkdirSync } = require('fs');
|
||||
const { join } = require('path');
|
||||
const { homedir } = require('os');
|
||||
|
||||
const CODEGRAPH_DIR = join(homedir(), '.codegraph');
|
||||
const MODELS_DIR = join(CODEGRAPH_DIR, 'models');
|
||||
const MODEL_ID = 'nomic-ai/nomic-embed-text-v1.5';
|
||||
|
||||
async function downloadModel() {
|
||||
// Ensure directories exist
|
||||
if (!existsSync(CODEGRAPH_DIR)) {
|
||||
mkdirSync(CODEGRAPH_DIR, { recursive: true });
|
||||
}
|
||||
if (!existsSync(MODELS_DIR)) {
|
||||
mkdirSync(MODELS_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// Check if model is already cached
|
||||
const modelCachePath = join(MODELS_DIR, MODEL_ID.replace('/', '/'));
|
||||
if (existsSync(modelCachePath)) {
|
||||
console.log('Embedding model already downloaded.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Downloading embedding model (~130MB)...');
|
||||
console.log('This is a one-time download for semantic code search.\n');
|
||||
|
||||
try {
|
||||
// Dynamic import for @xenova/transformers (ESM-only package)
|
||||
const { pipeline, env } = await import('@xenova/transformers');
|
||||
|
||||
// Configure cache directory
|
||||
env.cacheDir = MODELS_DIR;
|
||||
|
||||
// Download with progress
|
||||
await pipeline('feature-extraction', MODEL_ID, {
|
||||
progress_callback: (progress) => {
|
||||
if (progress.status === 'progress' && progress.file && progress.progress !== undefined) {
|
||||
const fileName = progress.file.split('/').pop();
|
||||
const percent = Math.round(progress.progress);
|
||||
process.stdout.write(`\rDownloading ${fileName}... ${percent}% `);
|
||||
} else if (progress.status === 'done') {
|
||||
process.stdout.write('\n');
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
console.log('\nEmbedding model ready!');
|
||||
} catch (error) {
|
||||
// Don't fail the install if model download fails
|
||||
// User can still use codegraph without semantic search
|
||||
console.log('\nNote: Could not download embedding model.');
|
||||
console.log('Semantic search will download it on first use.');
|
||||
if (process.env.DEBUG) {
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
downloadModel().catch(() => {
|
||||
// Silent exit - don't break npm install
|
||||
process.exit(0);
|
||||
});
|
||||
@@ -1078,69 +1078,6 @@ program
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* codegraph visualize [path]
|
||||
*/
|
||||
program
|
||||
.command('visualize [path]')
|
||||
.description('Open interactive graph visualization in your browser')
|
||||
.option('-p, --port <port>', 'Port to listen on (default: auto)', parseInt)
|
||||
.option('--no-open', 'Do not open browser automatically')
|
||||
.action(async (pathArg: string | undefined, options: { port?: number; open?: boolean }) => {
|
||||
const projectPath = resolveProjectPath(pathArg);
|
||||
|
||||
try {
|
||||
if (!isInitialized(projectPath)) {
|
||||
error(`CodeGraph not initialized in ${projectPath}`);
|
||||
info('Run "codegraph init -i" first');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const { default: CodeGraph } = await loadCodeGraph();
|
||||
const cg = await CodeGraph.open(projectPath);
|
||||
const stats = cg.getStats();
|
||||
|
||||
console.log(chalk.bold('\n CodeGraph Explorer\n'));
|
||||
info(`Project: ${projectPath}`);
|
||||
info(`Indexed: ${formatNumber(stats.nodeCount)} nodes, ${formatNumber(stats.edgeCount)} edges, ${formatNumber(stats.fileCount)} files\n`);
|
||||
|
||||
const { VisualizerServer } = await import('../visualizer/server');
|
||||
const server = new VisualizerServer(cg);
|
||||
const { url } = await server.start({ port: options.port, openBrowser: options.open !== false });
|
||||
|
||||
success(`Visualizer running at ${chalk.cyan(url)}`);
|
||||
console.log(chalk.dim(' Press Ctrl+C to stop\n'));
|
||||
|
||||
// Open browser
|
||||
if (options.open !== false) {
|
||||
const openCmd = process.platform === 'darwin' ? 'open' :
|
||||
process.platform === 'win32' ? 'start' : 'xdg-open';
|
||||
spawn(openCmd, [url], { detached: true, stdio: 'ignore' }).unref();
|
||||
}
|
||||
|
||||
// Handle shutdown — force exit on second Ctrl+C
|
||||
let shuttingDown = false;
|
||||
const shutdown = () => {
|
||||
if (shuttingDown) {
|
||||
process.exit(1);
|
||||
}
|
||||
shuttingDown = true;
|
||||
console.log(chalk.dim('\n Shutting down...'));
|
||||
server.stop().then(() => {
|
||||
cg.close();
|
||||
process.exit(0);
|
||||
}).catch(() => process.exit(1));
|
||||
// Force exit after 2s if graceful shutdown hangs
|
||||
setTimeout(() => process.exit(1), 2000).unref();
|
||||
};
|
||||
process.on('SIGINT', shutdown);
|
||||
process.on('SIGTERM', shutdown);
|
||||
} catch (err) {
|
||||
error(`Failed to start visualizer: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* codegraph mark-dirty [path]
|
||||
*
|
||||
|
||||
+6
-33
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Context Builder
|
||||
*
|
||||
* Builds rich context for tasks by combining semantic search with graph traversal.
|
||||
* Builds rich context for tasks by combining FTS search with graph traversal.
|
||||
* Outputs structured context ready to inject into Claude.
|
||||
*/
|
||||
|
||||
@@ -22,7 +22,6 @@ import {
|
||||
} from '../types';
|
||||
import { QueryBuilder } from '../db/queries';
|
||||
import { GraphTraverser } from '../graph';
|
||||
import { VectorManager } from '../vectors';
|
||||
import { formatContextAsMarkdown, formatContextAsJson } from './formatter';
|
||||
import { logDebug } from '../errors';
|
||||
import { validatePathWithinRoot } from '../utils';
|
||||
@@ -185,18 +184,15 @@ export class ContextBuilder {
|
||||
private projectRoot: string;
|
||||
private queries: QueryBuilder;
|
||||
private traverser: GraphTraverser;
|
||||
private vectorManager: VectorManager | null;
|
||||
|
||||
constructor(
|
||||
projectRoot: string,
|
||||
queries: QueryBuilder,
|
||||
traverser: GraphTraverser,
|
||||
vectorManager: VectorManager | null
|
||||
traverser: GraphTraverser
|
||||
) {
|
||||
this.projectRoot = projectRoot;
|
||||
this.queries = queries;
|
||||
this.traverser = traverser;
|
||||
this.vectorManager = vectorManager;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -394,21 +390,7 @@ export class ContextBuilder {
|
||||
exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 3));
|
||||
}
|
||||
|
||||
// Step 3: Try semantic search if vector manager is available
|
||||
let semanticResults: SearchResult[] = [];
|
||||
if (this.vectorManager && this.vectorManager.isInitialized()) {
|
||||
try {
|
||||
semanticResults = await this.vectorManager.search(query, {
|
||||
limit: opts.searchLimit,
|
||||
kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined,
|
||||
});
|
||||
logDebug('Semantic search results', { count: semanticResults.length });
|
||||
} catch (error) {
|
||||
logDebug('Semantic search failed, falling back to text search', { query, error: String(error) });
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Always run text search for natural language term matching
|
||||
// Step 3: Run text search for natural language term matching
|
||||
// This catches file-name and node-name matches that semantic search may miss,
|
||||
// which is critical for template-heavy codebases (e.g., Liquid/Shopify themes)
|
||||
// where file names are the primary identifiers.
|
||||
@@ -457,7 +439,7 @@ export class ContextBuilder {
|
||||
logDebug('Text search failed', { query, error: String(error) });
|
||||
}
|
||||
|
||||
// Step 5: Merge results, prioritizing exact matches, then text (path-boosted), then semantic
|
||||
// Step 4: Merge results, prioritizing exact matches, then text (path-boosted)
|
||||
const seenIds = new Set<string>();
|
||||
let searchResults: SearchResult[] = [];
|
||||
|
||||
@@ -477,14 +459,6 @@ export class ContextBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
// Add semantic results
|
||||
for (const result of semanticResults) {
|
||||
if (!seenIds.has(result.node.id)) {
|
||||
seenIds.add(result.node.id);
|
||||
searchResults.push(result);
|
||||
}
|
||||
}
|
||||
|
||||
const queryLower = query.toLowerCase();
|
||||
const isTestQuery = queryLower.includes('test') || queryLower.includes('spec');
|
||||
|
||||
@@ -1121,10 +1095,9 @@ export class ContextBuilder {
|
||||
export function createContextBuilder(
|
||||
projectRoot: string,
|
||||
queries: QueryBuilder,
|
||||
traverser: GraphTraverser,
|
||||
vectorManager: VectorManager | null
|
||||
traverser: GraphTraverser
|
||||
): ContextBuilder {
|
||||
return new ContextBuilder(projectRoot, queries, traverser, vectorManager);
|
||||
return new ContextBuilder(projectRoot, queries, traverser);
|
||||
}
|
||||
|
||||
// Re-export formatter
|
||||
|
||||
@@ -1290,7 +1290,6 @@ export class QueryBuilder {
|
||||
this.nodeCache.clear();
|
||||
this.db.transaction(() => {
|
||||
this.db.exec('DELETE FROM unresolved_refs');
|
||||
this.db.exec('DELETE FROM vectors');
|
||||
this.db.exec('DELETE FROM edges');
|
||||
this.db.exec('DELETE FROM nodes');
|
||||
this.db.exec('DELETE FROM files');
|
||||
|
||||
@@ -140,22 +140,6 @@ CREATE INDEX IF NOT EXISTS idx_unresolved_file_path ON unresolved_refs(file_path
|
||||
CREATE INDEX IF NOT EXISTS idx_unresolved_from_name ON unresolved_refs(from_node_id, reference_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_edges_provenance ON edges(provenance);
|
||||
|
||||
-- =============================================================================
|
||||
-- Vector Storage (for future semantic search)
|
||||
-- =============================================================================
|
||||
|
||||
-- Vector embeddings for semantic search
|
||||
-- Note: No foreign key constraint to allow standalone vector testing
|
||||
-- The VectorManager handles node-vector relationship at the application level
|
||||
CREATE TABLE IF NOT EXISTS vectors (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
embedding BLOB NOT NULL, -- Float32 array stored as blob
|
||||
model TEXT NOT NULL, -- Model used to generate embedding
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_vectors_model ON vectors(model);
|
||||
|
||||
-- Project metadata for version/provenance tracking
|
||||
CREATE TABLE IF NOT EXISTS project_metadata (
|
||||
key TEXT PRIMARY KEY,
|
||||
|
||||
+2
-124
@@ -46,7 +46,6 @@ import {
|
||||
ResolutionResult,
|
||||
} from './resolution';
|
||||
import { GraphTraverser, GraphQueryManager } from './graph';
|
||||
import { VectorManager, createVectorManager, EmbeddingProgress } from './vectors';
|
||||
import { ContextBuilder, createContextBuilder } from './context';
|
||||
import { Mutex, FileLock } from './utils';
|
||||
|
||||
@@ -63,7 +62,6 @@ export {
|
||||
export { IndexProgress, IndexResult, SyncResult } from './extraction';
|
||||
export { detectLanguage, isLanguageSupported, isGrammarLoaded, getSupportedLanguages, initGrammars, loadGrammarsForLanguages, loadAllGrammars } from './extraction';
|
||||
export { ResolutionResult } from './resolution';
|
||||
export { EmbeddingProgress } from './vectors';
|
||||
export {
|
||||
CodeGraphError,
|
||||
FileError,
|
||||
@@ -134,7 +132,6 @@ export class CodeGraph {
|
||||
private resolver: ReferenceResolver;
|
||||
private graphManager: GraphQueryManager;
|
||||
private traverser: GraphTraverser;
|
||||
private vectorManager: VectorManager | null = null;
|
||||
private contextBuilder: ContextBuilder;
|
||||
|
||||
// Mutex for preventing concurrent indexing operations (in-process)
|
||||
@@ -160,14 +157,10 @@ export class CodeGraph {
|
||||
this.resolver = createResolver(projectRoot, queries);
|
||||
this.graphManager = new GraphQueryManager(queries);
|
||||
this.traverser = new GraphTraverser(queries);
|
||||
// Vector manager — always created, embeddings generated lazily on first use
|
||||
this.vectorManager = createVectorManager(db.getDb(), queries, {});
|
||||
// Context builder (uses vector manager for semantic search)
|
||||
this.contextBuilder = createContextBuilder(
|
||||
projectRoot,
|
||||
queries,
|
||||
this.traverser,
|
||||
this.vectorManager
|
||||
this.traverser
|
||||
);
|
||||
}
|
||||
|
||||
@@ -328,11 +321,6 @@ export class CodeGraph {
|
||||
close(): void {
|
||||
// Release file lock if held
|
||||
this.fileLock.release();
|
||||
// Dispose vector manager first to release ONNX workers
|
||||
if (this.vectorManager) {
|
||||
this.vectorManager.dispose();
|
||||
this.vectorManager = null;
|
||||
}
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
@@ -838,102 +826,6 @@ export class CodeGraph {
|
||||
return this.graphManager.getNodeMetrics(nodeId);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Semantic Search (Vector Embeddings)
|
||||
// ===========================================================================
|
||||
|
||||
/**
|
||||
* Initialize the embedding system
|
||||
*
|
||||
* This downloads the embedding model on first use and initializes
|
||||
* the vector search system. Must be called before using semantic search.
|
||||
*/
|
||||
async initializeEmbeddings(): Promise<void> {
|
||||
if (!this.vectorManager) {
|
||||
this.vectorManager = createVectorManager(this.db.getDb(), this.queries, {
|
||||
embedder: {
|
||||
showProgress: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
await this.vectorManager.initialize();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if embeddings are initialized
|
||||
*/
|
||||
isEmbeddingsInitialized(): boolean {
|
||||
return this.vectorManager?.isInitialized() ?? false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embeddings for all eligible nodes
|
||||
*
|
||||
* @param onProgress - Optional progress callback
|
||||
* @returns Number of nodes embedded
|
||||
*/
|
||||
async generateEmbeddings(
|
||||
onProgress?: (progress: EmbeddingProgress) => void
|
||||
): Promise<number> {
|
||||
if (!this.vectorManager) {
|
||||
await this.initializeEmbeddings();
|
||||
}
|
||||
return this.vectorManager!.embedAllNodes(onProgress);
|
||||
}
|
||||
|
||||
/**
|
||||
* Semantic search using embeddings
|
||||
*
|
||||
* Searches for code nodes semantically similar to the query.
|
||||
* Requires embeddings to be initialized first.
|
||||
*
|
||||
* @param query - Natural language search query
|
||||
* @param limit - Maximum number of results (default: 10)
|
||||
* @returns Array of search results with similarity scores
|
||||
*/
|
||||
async semanticSearch(query: string, limit: number = 10): Promise<SearchResult[]> {
|
||||
if (!this.vectorManager || !this.vectorManager.isInitialized()) {
|
||||
throw new Error(
|
||||
'Embeddings not initialized. Call initializeEmbeddings() first.'
|
||||
);
|
||||
}
|
||||
return this.vectorManager.search(query, { limit });
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar code blocks
|
||||
*
|
||||
* Finds nodes semantically similar to a given node.
|
||||
* Requires embeddings to be initialized first.
|
||||
*
|
||||
* @param nodeId - ID of the node to find similar nodes for
|
||||
* @param limit - Maximum number of results (default: 10)
|
||||
* @returns Array of similar nodes with similarity scores
|
||||
*/
|
||||
async findSimilar(nodeId: string, limit: number = 10): Promise<SearchResult[]> {
|
||||
if (!this.vectorManager || !this.vectorManager.isInitialized()) {
|
||||
throw new Error(
|
||||
'Embeddings not initialized. Call initializeEmbeddings() first.'
|
||||
);
|
||||
}
|
||||
return this.vectorManager.findSimilar(nodeId, { limit });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get vector embedding statistics
|
||||
*/
|
||||
getEmbeddingStats(): {
|
||||
totalVectors: number;
|
||||
vssEnabled: boolean;
|
||||
modelId: string;
|
||||
dimension: number;
|
||||
} | null {
|
||||
if (!this.vectorManager) {
|
||||
return null;
|
||||
}
|
||||
return this.vectorManager.getStats();
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Context Building
|
||||
// ===========================================================================
|
||||
@@ -964,13 +856,6 @@ export class CodeGraph {
|
||||
query: string,
|
||||
options?: FindRelevantContextOptions
|
||||
): Promise<Subgraph> {
|
||||
// Update context builder with current vector manager
|
||||
this.contextBuilder = createContextBuilder(
|
||||
this.projectRoot,
|
||||
this.queries,
|
||||
this.traverser,
|
||||
this.vectorManager
|
||||
);
|
||||
return this.contextBuilder.findRelevantContext(query, options);
|
||||
}
|
||||
|
||||
@@ -978,7 +863,7 @@ export class CodeGraph {
|
||||
* Build context for a task
|
||||
*
|
||||
* Creates comprehensive context by:
|
||||
* 1. Running semantic search to find entry points
|
||||
* 1. Running FTS search to find entry points
|
||||
* 2. Expanding the graph around entry points
|
||||
* 3. Extracting code blocks for key nodes
|
||||
* 4. Formatting output for Claude
|
||||
@@ -991,13 +876,6 @@ export class CodeGraph {
|
||||
input: TaskInput,
|
||||
options?: BuildContextOptions
|
||||
): Promise<TaskContext | string> {
|
||||
// Update context builder with current vector manager
|
||||
this.contextBuilder = createContextBuilder(
|
||||
this.projectRoot,
|
||||
this.queries,
|
||||
this.traverser,
|
||||
this.vectorManager
|
||||
);
|
||||
return this.contextBuilder.buildContext(input, options);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,410 +0,0 @@
|
||||
/**
|
||||
* Text Embedder
|
||||
*
|
||||
* Generates vector embeddings using the nomic-embed-text model via Transformers.js.
|
||||
* Uses ONNX runtime under the hood for fast local inference.
|
||||
*/
|
||||
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import { homedir } from 'os';
|
||||
|
||||
// Global model cache directory - uses codegraph's models directory for shared embedding models
|
||||
const GLOBAL_MODELS_DIR = path.join(homedir(), '.codegraph', 'models');
|
||||
|
||||
// Dynamic import for @xenova/transformers (ESM-only package)
|
||||
// We use dynamic import to support CommonJS builds
|
||||
let transformersModule: typeof import('@xenova/transformers') | null = null;
|
||||
|
||||
async function getTransformers() {
|
||||
if (!transformersModule) {
|
||||
transformersModule = await import('@xenova/transformers');
|
||||
}
|
||||
return transformersModule;
|
||||
}
|
||||
|
||||
// Type for the feature extraction pipeline
|
||||
type FeatureExtractionPipeline = any;
|
||||
|
||||
/**
|
||||
* Default model for embeddings
|
||||
* nomic-embed-text-v1.5 produces 384-dimensional embeddings
|
||||
*/
|
||||
export const DEFAULT_MODEL = 'nomic-ai/nomic-embed-text-v1.5';
|
||||
export const EMBEDDING_DIMENSION = 768; // nomic-embed-text-v1.5 uses 768 dimensions
|
||||
|
||||
/**
|
||||
* Options for the embedder
|
||||
*/
|
||||
export interface EmbedderOptions {
|
||||
/** Model ID to use (default: nomic-ai/nomic-embed-text-v1.5) */
|
||||
modelId?: string;
|
||||
|
||||
/** Directory to cache the model (default: ~/.codegraph/models) */
|
||||
cacheDir?: string;
|
||||
|
||||
/** Whether to show progress during model download */
|
||||
showProgress?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Text embedding result
|
||||
*/
|
||||
export interface EmbeddingResult {
|
||||
/** The embedding vector */
|
||||
embedding: Float32Array;
|
||||
|
||||
/** Dimension of the embedding */
|
||||
dimension: number;
|
||||
|
||||
/** Model used to generate the embedding */
|
||||
model: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch embedding result
|
||||
*/
|
||||
export interface BatchEmbeddingResult {
|
||||
/** Array of embeddings in same order as input */
|
||||
embeddings: Float32Array[];
|
||||
|
||||
/** Dimension of each embedding */
|
||||
dimension: number;
|
||||
|
||||
/** Model used to generate embeddings */
|
||||
model: string;
|
||||
|
||||
/** Processing time in milliseconds */
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Text Embedder using Transformers.js
|
||||
*
|
||||
* Uses the nomic-embed-text-v1.5 model to generate embeddings for code
|
||||
* and natural language queries.
|
||||
*/
|
||||
export class TextEmbedder {
|
||||
private modelId: string;
|
||||
private cacheDir: string;
|
||||
private pipeline: FeatureExtractionPipeline | null = null;
|
||||
private initialized = false;
|
||||
private showProgress: boolean;
|
||||
|
||||
constructor(options: EmbedderOptions = {}) {
|
||||
this.modelId = options.modelId || DEFAULT_MODEL;
|
||||
this.cacheDir = options.cacheDir || GLOBAL_MODELS_DIR;
|
||||
this.showProgress = options.showProgress ?? false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the embedder by loading the model
|
||||
*
|
||||
* This will download the model on first use if not already cached.
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Load transformers.js dynamically (ESM-only package)
|
||||
const { pipeline, env } = await getTransformers();
|
||||
|
||||
// Configure transformers.js to use local cache
|
||||
env.cacheDir = this.cacheDir;
|
||||
|
||||
// Ensure cache directory exists
|
||||
if (!fs.existsSync(this.cacheDir)) {
|
||||
fs.mkdirSync(this.cacheDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Disable remote model checking if model is already cached
|
||||
// This speeds up initialization significantly
|
||||
const modelCacheExists = fs.existsSync(
|
||||
path.join(this.cacheDir, this.modelId.replace('/', '--'))
|
||||
);
|
||||
if (modelCacheExists) {
|
||||
env.allowRemoteModels = false;
|
||||
}
|
||||
|
||||
// Load the pipeline with quantized model to reduce WASM memory pressure.
|
||||
// Quantized (int8/uint8) is ~4x smaller than FP32 with minimal quality loss.
|
||||
this.pipeline = await pipeline('feature-extraction', this.modelId, {
|
||||
quantized: true,
|
||||
progress_callback: this.showProgress
|
||||
? (progress: { status: string; file?: string; progress?: number }) => {
|
||||
if (progress.status === 'progress' && progress.file && progress.progress) {
|
||||
const pct = Math.round(progress.progress);
|
||||
process.stdout.write(`\rDownloading ${progress.file}: ${pct}%\x1b[K`);
|
||||
} else if (progress.status === 'done') {
|
||||
process.stdout.write('\n');
|
||||
}
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the embedder is initialized
|
||||
*/
|
||||
isInitialized(): boolean {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the model ID being used
|
||||
*/
|
||||
getModelId(): string {
|
||||
return this.modelId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the embedding dimension
|
||||
*/
|
||||
getDimension(): number {
|
||||
return EMBEDDING_DIMENSION;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding for a single text
|
||||
*
|
||||
* @param text - Text to embed
|
||||
* @returns Embedding result
|
||||
*/
|
||||
async embed(text: string): Promise<EmbeddingResult> {
|
||||
if (!this.initialized || !this.pipeline) {
|
||||
throw new Error('Embedder not initialized. Call initialize() first.');
|
||||
}
|
||||
|
||||
// Prepare text for nomic-embed-text (it expects specific prefixes)
|
||||
const preparedText = this.prepareText(text, 'document');
|
||||
|
||||
// Generate embedding
|
||||
const output = await this.pipeline(preparedText, {
|
||||
pooling: 'mean',
|
||||
normalize: true,
|
||||
});
|
||||
|
||||
// Extract the embedding array - handle various data formats
|
||||
const data = output.data as unknown;
|
||||
const embedding = this.toFloat32Array(data);
|
||||
|
||||
return {
|
||||
embedding,
|
||||
dimension: embedding.length,
|
||||
model: this.modelId,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding for a query (uses different prefix)
|
||||
*
|
||||
* @param query - Query text to embed
|
||||
* @returns Embedding result
|
||||
*/
|
||||
async embedQuery(query: string): Promise<EmbeddingResult> {
|
||||
if (!this.initialized || !this.pipeline) {
|
||||
throw new Error('Embedder not initialized. Call initialize() first.');
|
||||
}
|
||||
|
||||
// Prepare text for nomic-embed-text query
|
||||
const preparedText = this.prepareText(query, 'search_query');
|
||||
|
||||
// Generate embedding
|
||||
const output = await this.pipeline(preparedText, {
|
||||
pooling: 'mean',
|
||||
normalize: true,
|
||||
});
|
||||
|
||||
// Extract the embedding array - handle various data formats
|
||||
const data = output.data as unknown;
|
||||
const embedding = this.toFloat32Array(data);
|
||||
|
||||
return {
|
||||
embedding,
|
||||
dimension: embedding.length,
|
||||
model: this.modelId,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embeddings for multiple texts in a batch
|
||||
*
|
||||
* @param texts - Array of texts to embed
|
||||
* @param type - Type of text (document or search_query)
|
||||
* @returns Batch embedding result
|
||||
*/
|
||||
async embedBatch(
|
||||
texts: string[],
|
||||
type: 'document' | 'search_query' = 'document'
|
||||
): Promise<BatchEmbeddingResult> {
|
||||
if (!this.initialized || !this.pipeline) {
|
||||
throw new Error('Embedder not initialized. Call initialize() first.');
|
||||
}
|
||||
|
||||
if (texts.length === 0) {
|
||||
return {
|
||||
embeddings: [],
|
||||
dimension: EMBEDDING_DIMENSION,
|
||||
model: this.modelId,
|
||||
durationMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
// Prepare all texts
|
||||
const preparedTexts = texts.map((t) => this.prepareText(t, type));
|
||||
|
||||
// Generate embeddings
|
||||
const outputs = await this.pipeline(preparedTexts, {
|
||||
pooling: 'mean',
|
||||
normalize: true,
|
||||
});
|
||||
|
||||
// Extract embeddings
|
||||
const embeddings: Float32Array[] = [];
|
||||
const dims = outputs.dims as number[];
|
||||
const dimension = dims[1] ?? EMBEDDING_DIMENSION;
|
||||
const data = outputs.data as unknown;
|
||||
const flatData = this.toFloat32Array(data);
|
||||
|
||||
for (let i = 0; i < texts.length; i++) {
|
||||
const start = i * dimension;
|
||||
const end = start + dimension;
|
||||
embeddings.push(flatData.slice(start, end));
|
||||
}
|
||||
|
||||
return {
|
||||
embeddings,
|
||||
dimension,
|
||||
model: this.modelId,
|
||||
durationMs: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert various array formats to Float32Array
|
||||
*/
|
||||
private toFloat32Array(data: unknown): Float32Array {
|
||||
if (data instanceof Float32Array) {
|
||||
return data;
|
||||
}
|
||||
if (Array.isArray(data)) {
|
||||
return new Float32Array(data);
|
||||
}
|
||||
if (data && typeof data === 'object' && 'length' in data) {
|
||||
// Handle TypedArray-like objects
|
||||
const arr = data as ArrayLike<number>;
|
||||
return Float32Array.from(Array.from(arr));
|
||||
}
|
||||
throw new Error('Unsupported data format for embedding');
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare text for the nomic-embed-text model
|
||||
*
|
||||
* The model expects specific prefixes for different tasks:
|
||||
* - "search_document: " for documents to be searched
|
||||
* - "search_query: " for search queries
|
||||
*/
|
||||
private prepareText(text: string, type: 'document' | 'search_query'): string {
|
||||
// Truncate very long texts (model has a max token limit)
|
||||
const maxLength = 8192; // nomic-embed-text-v1.5 supports 8192 tokens
|
||||
const truncatedText = text.length > maxLength ? text.slice(0, maxLength) : text;
|
||||
|
||||
// Add appropriate prefix
|
||||
if (type === 'search_query') {
|
||||
return `search_query: ${truncatedText}`;
|
||||
} else {
|
||||
return `search_document: ${truncatedText}`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create text representation of a code node for embedding
|
||||
*
|
||||
* Combines name, signature, docstring, and code snippet into
|
||||
* a searchable text representation.
|
||||
*/
|
||||
static createNodeText(node: {
|
||||
name: string;
|
||||
kind: string;
|
||||
qualifiedName?: string;
|
||||
signature?: string;
|
||||
docstring?: string;
|
||||
filePath: string;
|
||||
}): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
// Add kind and name
|
||||
parts.push(`${node.kind}: ${node.name}`);
|
||||
|
||||
// Add qualified name if different from name
|
||||
if (node.qualifiedName && node.qualifiedName !== node.name) {
|
||||
parts.push(`path: ${node.qualifiedName}`);
|
||||
}
|
||||
|
||||
// Add file path
|
||||
parts.push(`file: ${node.filePath}`);
|
||||
|
||||
// Add signature if present
|
||||
if (node.signature) {
|
||||
parts.push(`signature: ${node.signature}`);
|
||||
}
|
||||
|
||||
// Add docstring if present
|
||||
if (node.docstring) {
|
||||
parts.push(`documentation: ${node.docstring}`);
|
||||
}
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute cosine similarity between two embeddings
|
||||
*/
|
||||
static cosineSimilarity(a: Float32Array, b: Float32Array): number {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Embeddings must have the same dimension');
|
||||
}
|
||||
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
const aVal = a[i]!;
|
||||
const bVal = b[i]!;
|
||||
dotProduct += aVal * bVal;
|
||||
normA += aVal * aVal;
|
||||
normB += bVal * bVal;
|
||||
}
|
||||
|
||||
normA = Math.sqrt(normA);
|
||||
normB = Math.sqrt(normB);
|
||||
|
||||
if (normA === 0 || normB === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return dotProduct / (normA * normB);
|
||||
}
|
||||
|
||||
/**
|
||||
* Release resources
|
||||
*/
|
||||
dispose(): void {
|
||||
this.pipeline = null;
|
||||
this.initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a text embedder instance
|
||||
*/
|
||||
export function createEmbedder(options?: EmbedderOptions): TextEmbedder {
|
||||
return new TextEmbedder(options);
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
/**
|
||||
* Vectors Module
|
||||
*
|
||||
* Provides text embedding and vector similarity search for semantic code search.
|
||||
*/
|
||||
|
||||
export {
|
||||
TextEmbedder,
|
||||
createEmbedder,
|
||||
DEFAULT_MODEL,
|
||||
EMBEDDING_DIMENSION,
|
||||
EmbedderOptions,
|
||||
EmbeddingResult,
|
||||
BatchEmbeddingResult,
|
||||
} from './embedder';
|
||||
|
||||
export {
|
||||
VectorSearchManager,
|
||||
createVectorSearch,
|
||||
VectorSearchOptions,
|
||||
} from './search';
|
||||
|
||||
export {
|
||||
VectorManager,
|
||||
createVectorManager,
|
||||
VectorManagerOptions,
|
||||
EmbeddingProgress,
|
||||
} from './manager';
|
||||
@@ -1,363 +0,0 @@
|
||||
/**
|
||||
* Vector Manager
|
||||
*
|
||||
* High-level manager that coordinates embedding generation and vector search.
|
||||
*/
|
||||
|
||||
import { SqliteDatabase } from '../db/sqlite-adapter';
|
||||
import { Node, SearchResult, SearchOptions } from '../types';
|
||||
import { TextEmbedder, createEmbedder, EmbedderOptions, EMBEDDING_DIMENSION } from './embedder';
|
||||
import { VectorSearchManager, createVectorSearch } from './search';
|
||||
import { QueryBuilder } from '../db/queries';
|
||||
|
||||
/**
|
||||
* Progress callback for embedding generation
|
||||
*/
|
||||
export interface EmbeddingProgress {
|
||||
/** Current node index */
|
||||
current: number;
|
||||
|
||||
/** Total nodes to embed */
|
||||
total: number;
|
||||
|
||||
/** Current node being embedded */
|
||||
nodeName?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for the vector manager
|
||||
*/
|
||||
export interface VectorManagerOptions {
|
||||
/** Embedder options */
|
||||
embedder?: EmbedderOptions;
|
||||
|
||||
/** Node kinds to embed (default: functions, methods, classes, interfaces) */
|
||||
nodeKinds?: Node['kind'][];
|
||||
|
||||
/** Batch size for embedding generation */
|
||||
batchSize?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default node kinds to embed
|
||||
*/
|
||||
const DEFAULT_NODE_KINDS: Node['kind'][] = [
|
||||
'function',
|
||||
'method',
|
||||
'class',
|
||||
'interface',
|
||||
'type_alias',
|
||||
'module',
|
||||
'component',
|
||||
];
|
||||
|
||||
/**
|
||||
* Vector Manager
|
||||
*
|
||||
* Provides high-level interface for semantic search:
|
||||
* - Generates embeddings for code nodes
|
||||
* - Stores embeddings in the database
|
||||
* - Performs semantic similarity search
|
||||
*/
|
||||
export class VectorManager {
|
||||
private embedder: TextEmbedder;
|
||||
private searchManager: VectorSearchManager;
|
||||
private queries: QueryBuilder;
|
||||
private nodeKinds: Node['kind'][];
|
||||
private batchSize: number;
|
||||
private initialized = false;
|
||||
|
||||
constructor(
|
||||
db: SqliteDatabase,
|
||||
queries: QueryBuilder,
|
||||
options: VectorManagerOptions = {}
|
||||
) {
|
||||
this.embedder = createEmbedder(options.embedder);
|
||||
this.searchManager = createVectorSearch(db, EMBEDDING_DIMENSION);
|
||||
this.queries = queries;
|
||||
this.nodeKinds = options.nodeKinds || DEFAULT_NODE_KINDS;
|
||||
this.batchSize = options.batchSize || 32;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the vector manager
|
||||
*
|
||||
* Loads the embedding model and initializes vector search.
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Initialize embedder (downloads model if needed)
|
||||
await this.embedder.initialize();
|
||||
|
||||
// Initialize vector search (loads sqlite-vss if available)
|
||||
await this.searchManager.initialize();
|
||||
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the vector manager is initialized
|
||||
*/
|
||||
isInitialized(): boolean {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embeddings for all eligible nodes
|
||||
*
|
||||
* @param onProgress - Optional progress callback
|
||||
* @returns Number of nodes embedded
|
||||
*/
|
||||
async embedAllNodes(onProgress?: (progress: EmbeddingProgress) => void): Promise<number> {
|
||||
if (!this.initialized) {
|
||||
throw new Error('VectorManager not initialized. Call initialize() first.');
|
||||
}
|
||||
|
||||
// Get all nodes that should be embedded
|
||||
const nodesToEmbed: Node[] = [];
|
||||
for (const kind of this.nodeKinds) {
|
||||
const nodes = this.queries.getNodesByKind(kind);
|
||||
nodesToEmbed.push(...nodes);
|
||||
}
|
||||
|
||||
// Filter out nodes that already have embeddings
|
||||
const existingIds = new Set(this.searchManager.getIndexedNodeIds());
|
||||
const newNodes = nodesToEmbed.filter((n) => !existingIds.has(n.id));
|
||||
|
||||
if (newNodes.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Process in batches
|
||||
let processed = 0;
|
||||
const model = this.embedder.getModelId();
|
||||
|
||||
for (let i = 0; i < newNodes.length; i += this.batchSize) {
|
||||
const batch = newNodes.slice(i, i + this.batchSize);
|
||||
|
||||
// Create text representations
|
||||
const texts = batch.map((node) => TextEmbedder.createNodeText(node));
|
||||
|
||||
// Generate embeddings
|
||||
const result = await this.embedder.embedBatch(texts, 'document');
|
||||
|
||||
// Store embeddings
|
||||
const entries: Array<{ nodeId: string; embedding: Float32Array }> = [];
|
||||
for (let idx = 0; idx < batch.length; idx++) {
|
||||
const node = batch[idx];
|
||||
const embedding = result.embeddings[idx];
|
||||
if (node && embedding) {
|
||||
entries.push({ nodeId: node.id, embedding });
|
||||
}
|
||||
}
|
||||
this.searchManager.storeVectorBatch(entries, model);
|
||||
|
||||
processed += batch.length;
|
||||
|
||||
// Report progress
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
current: processed,
|
||||
total: newNodes.length,
|
||||
nodeName: batch[batch.length - 1]?.name,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding for a single node
|
||||
*
|
||||
* @param node - Node to embed
|
||||
*/
|
||||
async embedNode(node: Node): Promise<void> {
|
||||
if (!this.initialized) {
|
||||
throw new Error('VectorManager not initialized. Call initialize() first.');
|
||||
}
|
||||
|
||||
const text = TextEmbedder.createNodeText(node);
|
||||
const result = await this.embedder.embed(text);
|
||||
this.searchManager.storeVector(node.id, result.embedding, result.model);
|
||||
}
|
||||
|
||||
/**
|
||||
* Semantic search for nodes matching a query
|
||||
*
|
||||
* @param query - Natural language query
|
||||
* @param options - Search options
|
||||
* @returns Array of search results with similarity scores
|
||||
*/
|
||||
async search(query: string, options: SearchOptions = {}): Promise<SearchResult[]> {
|
||||
if (!this.initialized) {
|
||||
throw new Error('VectorManager not initialized. Call initialize() first.');
|
||||
}
|
||||
|
||||
const { limit = 10, kinds } = options;
|
||||
|
||||
// Generate query embedding
|
||||
const queryResult = await this.embedder.embedQuery(query);
|
||||
|
||||
// Search for similar vectors
|
||||
const vectorResults = this.searchManager.search(queryResult.embedding, {
|
||||
limit: limit * 2, // Get more results to filter
|
||||
minScore: 0.3, // Minimum similarity threshold
|
||||
});
|
||||
|
||||
// Get nodes and filter by kind if specified
|
||||
const results: SearchResult[] = [];
|
||||
for (const vr of vectorResults) {
|
||||
const node = this.queries.getNodeById(vr.nodeId);
|
||||
if (!node) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Filter by node kind if specified
|
||||
if (kinds && kinds.length > 0 && !kinds.includes(node.kind)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
results.push({
|
||||
node,
|
||||
score: vr.score,
|
||||
});
|
||||
|
||||
if (results.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find nodes similar to a given node
|
||||
*
|
||||
* @param nodeId - ID of the node to find similar nodes for
|
||||
* @param options - Search options
|
||||
* @returns Array of similar nodes with similarity scores
|
||||
*/
|
||||
async findSimilar(nodeId: string, options: SearchOptions = {}): Promise<SearchResult[]> {
|
||||
if (!this.initialized) {
|
||||
throw new Error('VectorManager not initialized. Call initialize() first.');
|
||||
}
|
||||
|
||||
const { limit = 10, kinds } = options;
|
||||
|
||||
// Get the node's embedding
|
||||
let embedding = this.searchManager.getVector(nodeId);
|
||||
|
||||
// If no embedding exists, generate one
|
||||
if (!embedding) {
|
||||
const node = this.queries.getNodeById(nodeId);
|
||||
if (!node) {
|
||||
throw new Error(`Node not found: ${nodeId}`);
|
||||
}
|
||||
|
||||
await this.embedNode(node);
|
||||
embedding = this.searchManager.getVector(nodeId);
|
||||
|
||||
if (!embedding) {
|
||||
throw new Error(`Failed to generate embedding for node: ${nodeId}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Search for similar vectors (excluding the source node)
|
||||
const vectorResults = this.searchManager.search(embedding, {
|
||||
limit: limit + 1, // Get one extra to exclude the source
|
||||
minScore: 0.3,
|
||||
});
|
||||
|
||||
// Get nodes and filter
|
||||
const results: SearchResult[] = [];
|
||||
for (const vr of vectorResults) {
|
||||
// Skip the source node
|
||||
if (vr.nodeId === nodeId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const node = this.queries.getNodeById(vr.nodeId);
|
||||
if (!node) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Filter by node kind if specified
|
||||
if (kinds && kinds.length > 0 && !kinds.includes(node.kind)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
results.push({
|
||||
node,
|
||||
score: vr.score,
|
||||
});
|
||||
|
||||
if (results.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete embedding for a node
|
||||
*
|
||||
* @param nodeId - ID of the node
|
||||
*/
|
||||
deleteNodeEmbedding(nodeId: string): void {
|
||||
this.searchManager.deleteVector(nodeId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics about vector storage
|
||||
*/
|
||||
getStats(): {
|
||||
totalVectors: number;
|
||||
vssEnabled: boolean;
|
||||
modelId: string;
|
||||
dimension: number;
|
||||
} {
|
||||
return {
|
||||
totalVectors: this.searchManager.getVectorCount(),
|
||||
vssEnabled: this.searchManager.isVssEnabled(),
|
||||
modelId: this.embedder.getModelId(),
|
||||
dimension: this.embedder.getDimension(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all vectors
|
||||
*/
|
||||
clear(): void {
|
||||
this.searchManager.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild the VSS index
|
||||
*/
|
||||
rebuildIndex(): void {
|
||||
this.searchManager.rebuildVssIndex();
|
||||
}
|
||||
|
||||
/**
|
||||
* Release resources
|
||||
*/
|
||||
dispose(): void {
|
||||
this.embedder.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a vector manager
|
||||
*/
|
||||
export function createVectorManager(
|
||||
db: SqliteDatabase,
|
||||
queries: QueryBuilder,
|
||||
options?: VectorManagerOptions
|
||||
): VectorManager {
|
||||
return new VectorManager(db, queries, options);
|
||||
}
|
||||
@@ -1,472 +0,0 @@
|
||||
/**
|
||||
* Vector Search
|
||||
*
|
||||
* Provides vector similarity search using sqlite-vss extension.
|
||||
* Falls back to brute-force cosine similarity if sqlite-vss is not available.
|
||||
*/
|
||||
|
||||
import { SqliteDatabase } from '../db/sqlite-adapter';
|
||||
import { Node } from '../types';
|
||||
import { TextEmbedder, EMBEDDING_DIMENSION } from './embedder';
|
||||
|
||||
/**
|
||||
* Options for vector search
|
||||
*/
|
||||
export interface VectorSearchOptions {
|
||||
/** Maximum number of results to return */
|
||||
limit?: number;
|
||||
|
||||
/** Minimum similarity score (0-1) */
|
||||
minScore?: number;
|
||||
|
||||
/** Node kinds to filter results */
|
||||
nodeKinds?: Node['kind'][];
|
||||
}
|
||||
|
||||
/**
|
||||
* Vector Search Manager
|
||||
*
|
||||
* Handles vector storage and similarity search for semantic code search.
|
||||
*/
|
||||
export class VectorSearchManager {
|
||||
private db: SqliteDatabase;
|
||||
private vssEnabled = false;
|
||||
private embeddingDimension: number;
|
||||
|
||||
constructor(db: SqliteDatabase, dimension: number = EMBEDDING_DIMENSION) {
|
||||
this.db = db;
|
||||
this.embeddingDimension = dimension;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize vector search
|
||||
*
|
||||
* Attempts to load sqlite-vss extension. Falls back to brute-force
|
||||
* search if the extension is not available.
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
try {
|
||||
// Try to load sqlite-vss extension
|
||||
await this.loadVssExtension();
|
||||
this.vssEnabled = true;
|
||||
console.log('sqlite-vss extension loaded successfully');
|
||||
|
||||
// Create the VSS virtual table
|
||||
this.createVssTable();
|
||||
} catch (error) {
|
||||
// Fall back to brute-force search
|
||||
console.warn(
|
||||
'sqlite-vss extension not available, falling back to brute-force search:',
|
||||
error instanceof Error ? error.message : String(error)
|
||||
);
|
||||
this.vssEnabled = false;
|
||||
}
|
||||
|
||||
// Ensure the vectors table exists (for both VSS and fallback modes)
|
||||
this.ensureVectorsTable();
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the sqlite-vss extension
|
||||
*/
|
||||
private async loadVssExtension(): Promise<void> {
|
||||
try {
|
||||
// The sqlite-vss npm package provides functions to load extensions
|
||||
const vss = await import('sqlite-vss');
|
||||
|
||||
// Use the load function which loads both vector0 and vss0
|
||||
// VSS extension expects the raw better-sqlite3 Database instance
|
||||
if (typeof vss.load === 'function') {
|
||||
vss.load(this.db as any);
|
||||
} else if (typeof vss.default?.load === 'function') {
|
||||
vss.default.load(this.db as any);
|
||||
} else {
|
||||
throw new Error('sqlite-vss load function not found');
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to load sqlite-vss: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the VSS virtual table for vector search
|
||||
*/
|
||||
private createVssTable(): void {
|
||||
// Check if the table already exists
|
||||
const tableExists = this.db
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='vss_vectors'")
|
||||
.get();
|
||||
|
||||
if (!tableExists) {
|
||||
// Create VSS virtual table
|
||||
// vss0 is the vector search extension
|
||||
this.db.exec(`
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS vss_vectors USING vss0(
|
||||
embedding(${this.embeddingDimension})
|
||||
);
|
||||
`);
|
||||
|
||||
// Create mapping table to link VSS rowids to node IDs
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS vss_map (
|
||||
rowid INTEGER PRIMARY KEY,
|
||||
node_id TEXT NOT NULL UNIQUE
|
||||
);
|
||||
`);
|
||||
|
||||
// Create index on node_id
|
||||
this.db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_vss_map_node ON vss_map(node_id);
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the basic vectors table exists (for fallback mode)
|
||||
*/
|
||||
private ensureVectorsTable(): void {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS vectors (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
embedding BLOB NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if VSS extension is enabled
|
||||
*/
|
||||
isVssEnabled(): boolean {
|
||||
return this.vssEnabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a vector embedding for a node
|
||||
*
|
||||
* @param nodeId - ID of the node
|
||||
* @param embedding - Vector embedding
|
||||
* @param model - Model used to generate embedding
|
||||
*/
|
||||
storeVector(nodeId: string, embedding: Float32Array, model: string): void {
|
||||
const now = Date.now();
|
||||
|
||||
// Store in the vectors table (always, for persistence)
|
||||
const blob = Buffer.from(embedding.buffer);
|
||||
this.db
|
||||
.prepare(
|
||||
`
|
||||
INSERT OR REPLACE INTO vectors (node_id, embedding, model, created_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`
|
||||
)
|
||||
.run(nodeId, blob, model, now);
|
||||
|
||||
// Also store in VSS table if enabled
|
||||
if (this.vssEnabled) {
|
||||
this.storeInVss(nodeId, embedding);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store vector in VSS virtual table
|
||||
*/
|
||||
private storeInVss(nodeId: string, embedding: Float32Array): void {
|
||||
try {
|
||||
// Check if already exists
|
||||
const existing = this.db
|
||||
.prepare('SELECT rowid FROM vss_map WHERE node_id = ?')
|
||||
.get(nodeId) as { rowid: number } | undefined;
|
||||
|
||||
if (existing) {
|
||||
// Update existing vector
|
||||
const vectorJson = JSON.stringify(Array.from(embedding));
|
||||
this.db
|
||||
.prepare('UPDATE vss_vectors SET embedding = ? WHERE rowid = ?')
|
||||
.run(vectorJson, existing.rowid);
|
||||
} else {
|
||||
// Insert new vector - get max rowid and increment
|
||||
const maxRow = this.db
|
||||
.prepare('SELECT MAX(rowid) as max FROM vss_map')
|
||||
.get() as { max: number | null } | undefined;
|
||||
const newRowid = (maxRow?.max ?? 0) + 1;
|
||||
|
||||
const vectorJson = JSON.stringify(Array.from(embedding));
|
||||
this.db
|
||||
.prepare('INSERT INTO vss_vectors (rowid, embedding) VALUES (?, ?)')
|
||||
.run(newRowid, vectorJson);
|
||||
|
||||
// Map the rowid to node_id
|
||||
this.db
|
||||
.prepare('INSERT INTO vss_map (rowid, node_id) VALUES (?, ?)')
|
||||
.run(newRowid, nodeId);
|
||||
}
|
||||
} catch (error) {
|
||||
// VSS operations can fail for various reasons (dimension mismatch, etc.)
|
||||
// Fall back to brute-force search silently
|
||||
console.warn(
|
||||
'VSS storage failed, using brute-force search:',
|
||||
error instanceof Error ? error.message : String(error)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store multiple vectors in a batch
|
||||
*
|
||||
* @param entries - Array of node IDs and embeddings
|
||||
* @param model - Model used to generate embeddings
|
||||
*/
|
||||
storeVectorBatch(
|
||||
entries: Array<{ nodeId: string; embedding: Float32Array }>,
|
||||
model: string
|
||||
): void {
|
||||
const now = Date.now();
|
||||
|
||||
// Use a transaction for better performance
|
||||
this.db.transaction(() => {
|
||||
for (const entry of entries) {
|
||||
const blob = Buffer.from(entry.embedding.buffer);
|
||||
this.db
|
||||
.prepare(
|
||||
`
|
||||
INSERT OR REPLACE INTO vectors (node_id, embedding, model, created_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`
|
||||
)
|
||||
.run(entry.nodeId, blob, model, now);
|
||||
|
||||
if (this.vssEnabled) {
|
||||
this.storeInVss(entry.nodeId, entry.embedding);
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get vector for a node
|
||||
*
|
||||
* @param nodeId - ID of the node
|
||||
* @returns Embedding or null if not found
|
||||
*/
|
||||
getVector(nodeId: string): Float32Array | null {
|
||||
const row = this.db
|
||||
.prepare('SELECT embedding FROM vectors WHERE node_id = ?')
|
||||
.get(nodeId) as { embedding: Buffer } | undefined;
|
||||
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Float32Array(row.embedding.buffer.slice(
|
||||
row.embedding.byteOffset,
|
||||
row.embedding.byteOffset + row.embedding.byteLength
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete vector for a node
|
||||
*
|
||||
* @param nodeId - ID of the node
|
||||
*/
|
||||
deleteVector(nodeId: string): void {
|
||||
this.db.prepare('DELETE FROM vectors WHERE node_id = ?').run(nodeId);
|
||||
|
||||
if (this.vssEnabled) {
|
||||
// Get the rowid before deleting
|
||||
const mapping = this.db
|
||||
.prepare('SELECT rowid FROM vss_map WHERE node_id = ?')
|
||||
.get(nodeId) as { rowid: number } | undefined;
|
||||
|
||||
if (mapping) {
|
||||
this.db.prepare('DELETE FROM vss_vectors WHERE rowid = ?').run(mapping.rowid);
|
||||
this.db.prepare('DELETE FROM vss_map WHERE node_id = ?').run(nodeId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for similar vectors
|
||||
*
|
||||
* @param queryEmbedding - Query vector to search for
|
||||
* @param options - Search options
|
||||
* @returns Array of node IDs with similarity scores
|
||||
*/
|
||||
search(
|
||||
queryEmbedding: Float32Array,
|
||||
options: VectorSearchOptions = {}
|
||||
): Array<{ nodeId: string; score: number }> {
|
||||
const { limit = 10, minScore = 0 } = options;
|
||||
|
||||
if (this.vssEnabled) {
|
||||
return this.searchWithVss(queryEmbedding, limit, minScore);
|
||||
} else {
|
||||
return this.searchBruteForce(queryEmbedding, limit, minScore);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search using sqlite-vss KNN search
|
||||
*/
|
||||
private searchWithVss(
|
||||
queryEmbedding: Float32Array,
|
||||
limit: number,
|
||||
minScore: number
|
||||
): Array<{ nodeId: string; score: number }> {
|
||||
try {
|
||||
const vectorJson = JSON.stringify(Array.from(queryEmbedding));
|
||||
// Sanitize limit to prevent SQL injection (ensure it's a positive integer)
|
||||
const safeLimit = Math.max(1, Math.floor(limit));
|
||||
|
||||
// Use VSS KNN search
|
||||
// The distance is L2 (euclidean), we need to convert to similarity score
|
||||
// Note: sqlite-vss requires LIMIT to be a literal, not a parameter
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`
|
||||
SELECT m.node_id, v.distance
|
||||
FROM (
|
||||
SELECT rowid, distance
|
||||
FROM vss_vectors
|
||||
WHERE vss_search(embedding, ?)
|
||||
LIMIT ${safeLimit}
|
||||
) v
|
||||
JOIN vss_map m ON m.rowid = v.rowid
|
||||
`
|
||||
)
|
||||
.all(vectorJson) as Array<{ node_id: string; distance: number }>;
|
||||
|
||||
// Convert L2 distance to similarity score (1 / (1 + distance))
|
||||
return rows
|
||||
.map((row) => ({
|
||||
nodeId: row.node_id,
|
||||
score: 1 / (1 + row.distance),
|
||||
}))
|
||||
.filter((r) => r.score >= minScore);
|
||||
} catch (error) {
|
||||
// VSS search failed, fall back to brute force
|
||||
console.warn(
|
||||
'VSS search failed, using brute-force:',
|
||||
error instanceof Error ? error.message : String(error)
|
||||
);
|
||||
return this.searchBruteForce(queryEmbedding, limit, minScore);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Brute-force search using cosine similarity
|
||||
*/
|
||||
private searchBruteForce(
|
||||
queryEmbedding: Float32Array,
|
||||
limit: number,
|
||||
minScore: number
|
||||
): Array<{ nodeId: string; score: number }> {
|
||||
// Get all vectors
|
||||
const rows = this.db
|
||||
.prepare('SELECT node_id, embedding FROM vectors')
|
||||
.all() as Array<{ node_id: string; embedding: Buffer }>;
|
||||
|
||||
// Calculate cosine similarity for each
|
||||
const results: Array<{ nodeId: string; score: number }> = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const embedding = new Float32Array(row.embedding.buffer.slice(
|
||||
row.embedding.byteOffset,
|
||||
row.embedding.byteOffset + row.embedding.byteLength
|
||||
));
|
||||
|
||||
const score = TextEmbedder.cosineSimilarity(queryEmbedding, embedding);
|
||||
|
||||
if (score >= minScore) {
|
||||
results.push({ nodeId: row.node_id, score });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by score descending and limit
|
||||
results.sort((a, b) => b.score - a.score);
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of stored vectors
|
||||
*/
|
||||
getVectorCount(): number {
|
||||
const result = this.db
|
||||
.prepare('SELECT COUNT(*) as count FROM vectors')
|
||||
.get() as { count: number };
|
||||
return result.count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a node has a vector
|
||||
*/
|
||||
hasVector(nodeId: string): boolean {
|
||||
const result = this.db
|
||||
.prepare('SELECT 1 FROM vectors WHERE node_id = ? LIMIT 1')
|
||||
.get(nodeId);
|
||||
return !!result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all node IDs that have vectors
|
||||
*/
|
||||
getIndexedNodeIds(): string[] {
|
||||
const rows = this.db
|
||||
.prepare('SELECT node_id FROM vectors')
|
||||
.all() as Array<{ node_id: string }>;
|
||||
return rows.map((r) => r.node_id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all vectors
|
||||
*/
|
||||
clear(): void {
|
||||
this.db.prepare('DELETE FROM vectors').run();
|
||||
|
||||
if (this.vssEnabled) {
|
||||
this.db.prepare('DELETE FROM vss_vectors').run();
|
||||
this.db.prepare('DELETE FROM vss_map').run();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild VSS index from vectors table
|
||||
*
|
||||
* Useful after bulk operations or if VSS index gets out of sync.
|
||||
*/
|
||||
rebuildVssIndex(): void {
|
||||
if (!this.vssEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Clear VSS tables
|
||||
this.db.prepare('DELETE FROM vss_vectors').run();
|
||||
this.db.prepare('DELETE FROM vss_map').run();
|
||||
|
||||
// Reload from vectors table
|
||||
const rows = this.db
|
||||
.prepare('SELECT node_id, embedding FROM vectors')
|
||||
.all() as Array<{ node_id: string; embedding: Buffer }>;
|
||||
|
||||
this.db.transaction(() => {
|
||||
for (const row of rows) {
|
||||
const embedding = new Float32Array(row.embedding.buffer.slice(
|
||||
row.embedding.byteOffset,
|
||||
row.embedding.byteOffset + row.embedding.byteLength
|
||||
));
|
||||
this.storeInVss(row.node_id, embedding);
|
||||
}
|
||||
})();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a vector search manager
|
||||
*/
|
||||
export function createVectorSearch(
|
||||
db: SqliteDatabase,
|
||||
dimension?: number
|
||||
): VectorSearchManager {
|
||||
return new VectorSearchManager(db, dimension);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,521 +0,0 @@
|
||||
/**
|
||||
* CodeGraph Visualizer Server
|
||||
*
|
||||
* Lightweight HTTP server that serves the graph visualization UI
|
||||
* and exposes REST API endpoints for querying the CodeGraph database.
|
||||
*/
|
||||
|
||||
import * as http from 'http';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as url from 'url';
|
||||
import { execFile } from 'child_process';
|
||||
import type CodeGraph from '../index';
|
||||
import type { Node, Edge, NodeKind } from '../types';
|
||||
|
||||
export interface VisualizerOptions {
|
||||
/** Port to listen on (0 = auto-assign) */
|
||||
port?: number;
|
||||
/** Whether to open browser automatically */
|
||||
openBrowser?: boolean;
|
||||
/** Host to bind to */
|
||||
host?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize a Subgraph (which uses Map) to plain JSON
|
||||
*/
|
||||
function serializeSubgraph(subgraph: { nodes: Map<string, Node>; edges: Edge[]; roots: string[] }) {
|
||||
return {
|
||||
nodes: Array.from(subgraph.nodes.values()),
|
||||
edges: subgraph.edges,
|
||||
roots: subgraph.roots,
|
||||
};
|
||||
}
|
||||
|
||||
export class VisualizerServer {
|
||||
private cg: CodeGraph;
|
||||
private server: http.Server | null = null;
|
||||
private projectRoot: string;
|
||||
private symbolIndexCache: string | null = null;
|
||||
private claudeAvailable: boolean | null = null;
|
||||
|
||||
constructor(cg: CodeGraph) {
|
||||
this.cg = cg;
|
||||
this.projectRoot = cg.getProjectRoot();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a compact symbol index string for Claude prompts
|
||||
*/
|
||||
private buildSymbolIndex(): string {
|
||||
if (this.symbolIndexCache) return this.symbolIndexCache;
|
||||
|
||||
const validKinds: NodeKind[] = ['function', 'method', 'class', 'interface', 'component', 'route', 'enum', 'type_alias'];
|
||||
const byFile = new Map<string, string[]>();
|
||||
|
||||
for (const kind of validKinds) {
|
||||
for (const node of this.cg.getNodesByKind(kind)) {
|
||||
const symbols = byFile.get(node.filePath) || [];
|
||||
symbols.push(`${node.kind}:${node.name}`);
|
||||
byFile.set(node.filePath, symbols);
|
||||
}
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
for (const [file, symbols] of byFile) {
|
||||
lines.push(`${file}: ${symbols.join(', ')}`);
|
||||
}
|
||||
|
||||
this.symbolIndexCache = lines.join('\n');
|
||||
return this.symbolIndexCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask Claude CLI to interpret a natural language question into relevant symbol names
|
||||
*/
|
||||
private async askClaude(question: string): Promise<string[] | null> {
|
||||
// Check if claude is available (cache result)
|
||||
if (this.claudeAvailable === false) return null;
|
||||
|
||||
const symbolIndex = this.buildSymbolIndex();
|
||||
|
||||
const prompt = `Given the question and codebase symbol index below, identify the single best ENTRY POINT symbol — the one function, component, or route handler where this flow starts.
|
||||
|
||||
Rules:
|
||||
- Pick ONE symbol that is the starting point a user or request would hit first
|
||||
- Prefer page components, route handlers, or top-level functions
|
||||
- Do NOT pick utility functions, helpers, or middleware
|
||||
|
||||
Return ONLY this JSON, nothing else:
|
||||
{"entry": "symbolName"}
|
||||
|
||||
Question: "${question}"
|
||||
|
||||
Symbol index:
|
||||
${symbolIndex}`;
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const timeout = setTimeout(() => {
|
||||
resolve(null);
|
||||
}, 30000);
|
||||
|
||||
execFile('claude', ['-p', prompt, '--output-format', 'text'], {
|
||||
timeout: 30000,
|
||||
maxBuffer: 1024 * 1024,
|
||||
}, (err, stdout) => {
|
||||
clearTimeout(timeout);
|
||||
|
||||
if (err) {
|
||||
this.claudeAvailable = false;
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
this.claudeAvailable = true;
|
||||
|
||||
// Parse Claude's response — try object format first, then array fallback
|
||||
try {
|
||||
const text = stdout.trim();
|
||||
// Try to extract JSON object {"entry": ..., "flow": [...]}
|
||||
const objMatch = text.match(/\{[\s\S]*\}/);
|
||||
if (objMatch) {
|
||||
const parsed = JSON.parse(objMatch[0]) as { entry?: string; flow?: string[] };
|
||||
if (parsed.flow && Array.isArray(parsed.flow) && parsed.flow.length > 0) {
|
||||
// Return flow with entry first
|
||||
const names = parsed.flow.map(String);
|
||||
if (parsed.entry && !names.includes(parsed.entry)) {
|
||||
names.unshift(String(parsed.entry));
|
||||
}
|
||||
resolve(names);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Fallback: try JSON array
|
||||
const arrMatch = text.match(/\[[\s\S]*\]/);
|
||||
if (arrMatch) {
|
||||
const names = JSON.parse(arrMatch[0]) as string[];
|
||||
if (Array.isArray(names) && names.length > 0) {
|
||||
resolve(names.map(String));
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Parse failed
|
||||
}
|
||||
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the visualizer server
|
||||
*/
|
||||
async start(options: VisualizerOptions = {}): Promise<{ port: number; url: string }> {
|
||||
const host = options.host || '127.0.0.1';
|
||||
const port = options.port || 0;
|
||||
|
||||
this.server = http.createServer((req, res) => {
|
||||
this.handleRequest(req, res).catch((err) => {
|
||||
console.error('[Visualizer] Request error:', err);
|
||||
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Internal server error' }));
|
||||
});
|
||||
});
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.server!.listen(port, host, () => {
|
||||
const addr = this.server!.address();
|
||||
if (!addr || typeof addr === 'string') {
|
||||
reject(new Error('Failed to get server address'));
|
||||
return;
|
||||
}
|
||||
const serverUrl = `http://${host}:${addr.port}`;
|
||||
resolve({ port: addr.port, url: serverUrl });
|
||||
});
|
||||
|
||||
this.server!.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the server
|
||||
*/
|
||||
stop(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (this.server) {
|
||||
this.server.close(() => resolve());
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
|
||||
const parsedUrl = url.parse(req.url || '/', true);
|
||||
const pathname = parsedUrl.pathname || '/';
|
||||
|
||||
// CORS headers for local development
|
||||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||||
res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
|
||||
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
|
||||
|
||||
if (req.method === 'OPTIONS') {
|
||||
res.writeHead(204);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
// API routes
|
||||
if (pathname.startsWith('/api/')) {
|
||||
return this.handleAPI(pathname, parsedUrl.query as Record<string, string>, res);
|
||||
}
|
||||
|
||||
// Static file serving
|
||||
return this.serveStatic(pathname, res);
|
||||
}
|
||||
|
||||
private async handleAPI(
|
||||
pathname: string,
|
||||
query: Record<string, string>,
|
||||
res: http.ServerResponse
|
||||
): Promise<void> {
|
||||
const json = (data: unknown, status = 200) => {
|
||||
res.writeHead(status, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(data));
|
||||
};
|
||||
|
||||
try {
|
||||
// GET /api/status
|
||||
if (pathname === '/api/status') {
|
||||
const stats = this.cg.getStats();
|
||||
json({ stats, projectRoot: this.projectRoot, projectName: path.basename(this.projectRoot) });
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/embeddings/status
|
||||
if (pathname === '/api/embeddings/status') {
|
||||
const embeddingStats = this.cg.getEmbeddingStats();
|
||||
const isInitialized = this.cg.isEmbeddingsInitialized();
|
||||
const totalVectors = embeddingStats?.totalVectors ?? 0;
|
||||
const stats = this.cg.getStats();
|
||||
// Consider ready if we have vectors for at least half the eligible nodes
|
||||
const eligibleNodes = stats.nodeCount - (stats.nodesByKind.file ?? 0) - (stats.nodesByKind.import ?? 0);
|
||||
const isReady = totalVectors > 0 && totalVectors >= eligibleNodes * 0.5;
|
||||
json({ isEnabled: true, isInitialized, isReady, totalVectors, eligibleNodes });
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/embeddings/generate — SSE stream that enables, initializes, and generates embeddings
|
||||
if (pathname === '/api/embeddings/generate') {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
});
|
||||
|
||||
const send = (event: string, data: unknown) => {
|
||||
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
||||
};
|
||||
|
||||
try {
|
||||
// Step 1: Initialize embedding model (downloads on first use)
|
||||
send('status', { phase: 'model', message: 'Loading embedding model (first time may download ~30MB)...' });
|
||||
await this.cg.initializeEmbeddings();
|
||||
send('status', { phase: 'model', message: 'Embedding model ready' });
|
||||
|
||||
// Step 3: Generate embeddings with progress
|
||||
send('status', { phase: 'embedding', message: 'Generating embeddings...' });
|
||||
const count = await this.cg.generateEmbeddings((progress) => {
|
||||
send('progress', {
|
||||
current: progress.current,
|
||||
total: progress.total,
|
||||
nodeName: progress.nodeName,
|
||||
percent: progress.total > 0 ? Math.round((progress.current / progress.total) * 100) : 0,
|
||||
});
|
||||
});
|
||||
|
||||
send('complete', { totalEmbedded: count, message: `Generated ${count} embeddings` });
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
send('error', { message });
|
||||
}
|
||||
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/search?q=...&kind=...&limit=...
|
||||
if (pathname === '/api/search') {
|
||||
const q = query.q || '';
|
||||
const kind = query.kind as NodeKind | undefined;
|
||||
const limit = parseInt(query.limit || '30', 10);
|
||||
if (!q) {
|
||||
json({ results: [] });
|
||||
return;
|
||||
}
|
||||
const results = this.cg.searchNodes(q, { kinds: kind ? [kind] : undefined, limit });
|
||||
json({ results });
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/explore?q=...
|
||||
// Find the best entry point, then return its call graph
|
||||
if (pathname === '/api/explore') {
|
||||
const q = query.q || '';
|
||||
if (!q) {
|
||||
json({ nodes: [], edges: [], roots: [], entryPoint: null });
|
||||
return;
|
||||
}
|
||||
|
||||
let entryNodeId: string | null = null;
|
||||
let usedClaude = false;
|
||||
const validKinds: NodeKind[] = ['function', 'method', 'class', 'interface', 'component', 'route'];
|
||||
|
||||
// Try Claude CLI to find the best entry point
|
||||
const claudeNames = await this.askClaude(q);
|
||||
if (claudeNames && claudeNames.length > 0) {
|
||||
usedClaude = true;
|
||||
// Find the entry point in the graph
|
||||
for (const name of claudeNames) {
|
||||
if (entryNodeId) break;
|
||||
const results = this.cg.searchNodes(name, { kinds: validKinds, limit: 3 });
|
||||
for (const r of results) {
|
||||
if (r.node.name.toLowerCase() === name.toLowerCase() ||
|
||||
r.node.name.toLowerCase().includes(name.toLowerCase()) ||
|
||||
name.toLowerCase().includes(r.node.name.toLowerCase())) {
|
||||
entryNodeId = r.node.id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keyword fallback: find best match from query keywords
|
||||
if (!entryNodeId) {
|
||||
const stopWords = new Set(['how', 'does', 'what', 'the', 'is', 'a', 'an', 'and', 'or', 'in', 'to', 'for', 'of', 'with', 'when', 'do', 'it', 'my', 'work', 'works', 'about', 'show', 'me']);
|
||||
const keywords = q.toLowerCase().split(/\s+/)
|
||||
.map(w => w.replace(/[^a-z0-9]/g, ''))
|
||||
.filter(w => w.length >= 2 && !stopWords.has(w));
|
||||
|
||||
for (const kw of keywords) {
|
||||
if (entryNodeId) break;
|
||||
const results = this.cg.searchNodes(kw, { kinds: validKinds, limit: 5 });
|
||||
if (results.length > 0) {
|
||||
entryNodeId = results[0]!.node.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!entryNodeId) {
|
||||
json({ nodes: [], edges: [], roots: [], entryPoint: null });
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the call graph from this entry point (depth 3)
|
||||
const callGraph = this.cg.getCallGraph(entryNodeId, 3);
|
||||
const result = serializeSubgraph(callGraph);
|
||||
|
||||
json({
|
||||
nodes: result.nodes,
|
||||
edges: result.edges,
|
||||
roots: [entryNodeId],
|
||||
entryPoint: entryNodeId,
|
||||
usedClaude,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/overview?limit=...
|
||||
if (pathname === '/api/overview') {
|
||||
const limit = parseInt(query.limit || '50', 10);
|
||||
// Get top-level exported classes, functions, components
|
||||
const kinds: NodeKind[] = ['class', 'function', 'interface', 'component', 'enum', 'type_alias'];
|
||||
const nodes: Node[] = [];
|
||||
for (const kind of kinds) {
|
||||
const kindNodes = this.cg.getNodesByKind(kind);
|
||||
for (const n of kindNodes) {
|
||||
if (n.isExported || n.kind === 'class' || n.kind === 'component') {
|
||||
nodes.push(n);
|
||||
}
|
||||
if (nodes.length >= limit) break;
|
||||
}
|
||||
if (nodes.length >= limit) break;
|
||||
}
|
||||
json({ nodes });
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/files
|
||||
if (pathname === '/api/files') {
|
||||
const files = this.cg.getFiles();
|
||||
json({ files });
|
||||
return;
|
||||
}
|
||||
|
||||
// Routes with node ID: /api/node/<id>/...
|
||||
const nodeMatch = pathname.match(/^\/api\/node\/([^/]+)(\/.*)?$/);
|
||||
if (nodeMatch) {
|
||||
const nodeId = decodeURIComponent(nodeMatch[1]!);
|
||||
const sub = nodeMatch[2] || '';
|
||||
|
||||
// GET /api/node/<id>
|
||||
if (!sub || sub === '/') {
|
||||
const node = this.cg.getNode(nodeId);
|
||||
if (!node) {
|
||||
json({ error: 'Node not found' }, 404);
|
||||
return;
|
||||
}
|
||||
const code = await this.cg.getCode(nodeId);
|
||||
const ancestors = this.cg.getAncestors(nodeId);
|
||||
json({ node, code, ancestors });
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/node/<id>/callers?depth=...
|
||||
if (sub === '/callers') {
|
||||
const depth = parseInt(query.depth || '1', 10);
|
||||
const items = this.cg.getCallers(nodeId, depth);
|
||||
json({ items });
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/node/<id>/callees?depth=...
|
||||
if (sub === '/callees') {
|
||||
const depth = parseInt(query.depth || '1', 10);
|
||||
const items = this.cg.getCallees(nodeId, depth);
|
||||
json({ items });
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/node/<id>/children
|
||||
if (sub === '/children') {
|
||||
const children = this.cg.getChildren(nodeId);
|
||||
json({ children });
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/node/<id>/impact?depth=...
|
||||
if (sub === '/impact') {
|
||||
const depth = parseInt(query.depth || '2', 10);
|
||||
const subgraph = this.cg.getImpactRadius(nodeId, depth);
|
||||
json(serializeSubgraph(subgraph));
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/node/<id>/callgraph?depth=...
|
||||
if (sub === '/callgraph') {
|
||||
const depth = parseInt(query.depth || '2', 10);
|
||||
const subgraph = this.cg.getCallGraph(nodeId, depth);
|
||||
json(serializeSubgraph(subgraph));
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/node/<id>/context
|
||||
if (sub === '/context') {
|
||||
const context = this.cg.getContext(nodeId);
|
||||
json({ context });
|
||||
return;
|
||||
}
|
||||
|
||||
json({ error: 'Unknown endpoint' }, 404);
|
||||
return;
|
||||
}
|
||||
|
||||
// GET /api/file-nodes?path=...
|
||||
if (pathname === '/api/file-nodes') {
|
||||
const filePath = query.path || '';
|
||||
if (!filePath) {
|
||||
json({ error: 'path parameter required' }, 400);
|
||||
return;
|
||||
}
|
||||
const nodes = this.cg.getNodesInFile(filePath);
|
||||
json({ nodes });
|
||||
return;
|
||||
}
|
||||
|
||||
json({ error: 'Unknown API endpoint' }, 404);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
json({ error: message }, 500);
|
||||
}
|
||||
}
|
||||
|
||||
private serveStatic(pathname: string, res: http.ServerResponse): void {
|
||||
if (pathname === '/' || pathname === '/index.html') {
|
||||
pathname = '/index.html';
|
||||
}
|
||||
|
||||
// Resolve from the public directory next to this file
|
||||
const publicDir = path.join(__dirname, 'public');
|
||||
const filePath = path.join(publicDir, pathname);
|
||||
|
||||
// Security: prevent directory traversal
|
||||
if (!filePath.startsWith(publicDir)) {
|
||||
res.writeHead(403);
|
||||
res.end('Forbidden');
|
||||
return;
|
||||
}
|
||||
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
const mimeTypes: Record<string, string> = {
|
||||
'.html': 'text/html',
|
||||
'.css': 'text/css',
|
||||
'.js': 'application/javascript',
|
||||
'.json': 'application/json',
|
||||
'.png': 'image/png',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.ico': 'image/x-icon',
|
||||
};
|
||||
|
||||
try {
|
||||
const content = fs.readFileSync(filePath);
|
||||
res.writeHead(200, { 'Content-Type': mimeTypes[ext] || 'application/octet-stream' });
|
||||
res.end(content);
|
||||
} catch {
|
||||
res.writeHead(404, { 'Content-Type': 'text/plain' });
|
||||
res.end('Not found');
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user