Merge pull request #70 from colbymchenry/feat/improved-search-tokenization

feat: Improve search tokenization with camelCase splitting

package.json

@@ -16,7 +16,7 @@
     "build": "tsc && npm run copy-assets",
     "postinstall": "node scripts/postinstall.js",
     "preuninstall": "node dist/bin/uninstall.js",
-    "copy-assets": "node -e \"const fs=require('fs');fs.mkdirSync('dist/db',{recursive:true});fs.copyFileSync('src/db/schema.sql','dist/db/schema.sql');fs.mkdirSync('dist/extraction/wasm',{recursive:true});fs.readdirSync('src/extraction/wasm').filter(f=>f.endsWith('.wasm')).forEach(f=>fs.copyFileSync('src/extraction/wasm/'+f,'dist/extraction/wasm/'+f))\"",
+    "copy-assets": "node -e \"const fs=require('fs');fs.mkdirSync('dist/db',{recursive:true});fs.copyFileSync('src/db/schema.sql','dist/db/schema.sql');fs.mkdirSync('dist/extraction/wasm',{recursive:true});fs.readdirSync('src/extraction/wasm').filter(f=>f.endsWith('.wasm')).forEach(f=>fs.copyFileSync('src/extraction/wasm/'+f,'dist/extraction/wasm/'+f));fs.mkdirSync('dist/visualizer/public',{recursive:true});fs.copyFileSync('src/visualizer/public/index.html','dist/visualizer/public/index.html')\"",
     "dev": "tsc --watch",
     "cli": "npm run build && node dist/bin/codegraph.js",
     "test": "vitest run",

src/bin/codegraph.ts

@@ -955,6 +955,70 @@ program
     }
   });
 
+/**
+ * codegraph visualize [path]
+ */
+program
+  .command('visualize [path]')
+  .description('Open interactive graph visualization in your browser')
+  .option('-p, --port <port>', 'Port to listen on (default: auto)', parseInt)
+  .option('--no-open', 'Do not open browser automatically')
+  .action(async (pathArg: string | undefined, options: { port?: number; open?: boolean }) => {
+    const projectPath = resolveProjectPath(pathArg);
+
+    try {
+      if (!isInitialized(projectPath)) {
+        error(`CodeGraph not initialized in ${projectPath}`);
+        info('Run "codegraph init -i" first');
+        process.exit(1);
+      }
+
+      const { default: CodeGraph } = await loadCodeGraph();
+      const cg = await CodeGraph.open(projectPath);
+      const stats = cg.getStats();
+
+      console.log(chalk.bold('\n  CodeGraph Explorer\n'));
+      info(`Project: ${projectPath}`);
+      info(`Indexed: ${formatNumber(stats.nodeCount)} nodes, ${formatNumber(stats.edgeCount)} edges, ${formatNumber(stats.fileCount)} files\n`);
+
+      const { VisualizerServer } = await import('../visualizer/server');
+      const server = new VisualizerServer(cg);
+      const { url } = await server.start({ port: options.port, openBrowser: options.open !== false });
+
+      success(`Visualizer running at ${chalk.cyan(url)}`);
+      console.log(chalk.dim('  Press Ctrl+C to stop\n'));
+
+      // Open browser
+      if (options.open !== false) {
+        const openCmd = process.platform === 'darwin' ? 'open' :
+                        process.platform === 'win32' ? 'start' : 'xdg-open';
+        spawn(openCmd, [url], { detached: true, stdio: 'ignore' }).unref();
+      }
+
+      // Handle shutdown — force exit on second Ctrl+C
+      let shuttingDown = false;
+      const shutdown = () => {
+        if (shuttingDown) {
+          process.exit(1);
+        }
+        shuttingDown = true;
+        console.log(chalk.dim('\n  Shutting down...'));
+        server.stop().then(() => {
+          cg.close();
+          process.exit(0);
+        }).catch(() => process.exit(1));
+        // Force exit after 2s if graceful shutdown hangs
+        setTimeout(() => process.exit(1), 2000).unref();
+      };
+      process.on('SIGINT', shutdown);
+      process.on('SIGTERM', shutdown);
+    } catch (err) {
+      captureException(err);
+      error(`Failed to start visualizer: ${err instanceof Error ? err.message : String(err)}`);
+      process.exit(1);
+    }
+  });
+
 /**
  * codegraph mark-dirty [path]
  *

src/search/query-utils.ts

@@ -8,9 +8,11 @@ import * as path from 'path';
 import { Node } from '../types';
 
 /**
- * Common stop words to filter from search queries
+ * Common stop words to filter from search queries.
+ * Includes generic English + code-specific noise words.
  */
 export const STOP_WORDS = new Set([
+  // English
   'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
   'of', 'with', 'by', 'from', 'is', 'it', 'that', 'this', 'are', 'was',
   'be', 'has', 'had', 'have', 'do', 'does', 'did', 'will', 'would', 'could',
@@ -18,17 +20,41 @@ export const STOP_WORDS = new Set([
   'every', 'how', 'what', 'where', 'when', 'who', 'which', 'why',
   'i', 'me', 'my', 'we', 'our', 'you', 'your', 'he', 'she', 'they',
   'find', 'show', 'get', 'list', 'give', 'tell',
+  'been', 'done', 'made', 'used', 'using', 'work', 'works', 'found',
+  'also', 'into', 'then', 'than', 'just', 'more', 'some', 'such',
+  'over', 'only', 'new', 'out', 'its', 'so', 'up', 'as', 'if',
+  // Code-specific noise
+  'code', 'file', 'files', 'function', 'method', 'class', 'type',
+  'build', 'run', 'test', 'fix', 'bug', 'call', 'called', 'set', 'add',
 ]);
 
 /**
- * Extract meaningful search terms from a natural language query
+ * Extract meaningful search terms from a natural language query.
+ * Splits camelCase, PascalCase, snake_case, SCREAMING_SNAKE, and dot.notation
+ * into individual tokens before filtering.
  */
 export function extractSearchTerms(query: string): string[] {
-  return query
+  const tokens = new Set<string>();
-    .toLowerCase()
+
-    .replace(/[^\w\s-]/g, ' ')
+  // Split camelCase / PascalCase: "getUserName" → "get User Name"
-    .split(/\s+/)
+  const camelSplit = query
-    .filter(term => term.length > 1 && !STOP_WORDS.has(term));
+    .replace(/([a-z])([A-Z])/g, '$1 $2')
+    .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2');
+
+  // Replace underscores and dots with spaces (snake_case, dot.notation)
+  const normalised = camelSplit.replace(/[_.]+/g, ' ');
+
+  // Split on any non-alphanumeric character
+  const words = normalised.split(/[^a-zA-Z0-9]+/).filter(Boolean);
+
+  for (const word of words) {
+    const lower = word.toLowerCase();
+    if (lower.length < 3) continue;
+    if (STOP_WORDS.has(lower)) continue;
+    tokens.add(lower);
+  }
+
+  return [...tokens];
 }
 
 /**

src/visualizer/server.ts

@@ -0,0 +1,530 @@
+/**
+ * CodeGraph Visualizer Server
+ *
+ * Lightweight HTTP server that serves the graph visualization UI
+ * and exposes REST API endpoints for querying the CodeGraph database.
+ */
+
+import * as http from 'http';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as url from 'url';
+import { execFile } from 'child_process';
+import type CodeGraph from '../index';
+import type { Node, Edge, NodeKind } from '../types';
+
+export interface VisualizerOptions {
+  /** Port to listen on (0 = auto-assign) */
+  port?: number;
+  /** Whether to open browser automatically */
+  openBrowser?: boolean;
+  /** Host to bind to */
+  host?: string;
+}
+
+/**
+ * Serialize a Subgraph (which uses Map) to plain JSON
+ */
+function serializeSubgraph(subgraph: { nodes: Map<string, Node>; edges: Edge[]; roots: string[] }) {
+  return {
+    nodes: Array.from(subgraph.nodes.values()),
+    edges: subgraph.edges,
+    roots: subgraph.roots,
+  };
+}
+
+export class VisualizerServer {
+  private cg: CodeGraph;
+  private server: http.Server | null = null;
+  private projectRoot: string;
+  private symbolIndexCache: string | null = null;
+  private claudeAvailable: boolean | null = null;
+
+  constructor(cg: CodeGraph) {
+    this.cg = cg;
+    this.projectRoot = cg.getProjectRoot();
+  }
+
+  /**
+   * Build a compact symbol index string for Claude prompts
+   */
+  private buildSymbolIndex(): string {
+    if (this.symbolIndexCache) return this.symbolIndexCache;
+
+    const validKinds: NodeKind[] = ['function', 'method', 'class', 'interface', 'component', 'route', 'enum', 'type_alias'];
+    const byFile = new Map<string, string[]>();
+
+    for (const kind of validKinds) {
+      for (const node of this.cg.getNodesByKind(kind)) {
+        const symbols = byFile.get(node.filePath) || [];
+        symbols.push(`${node.kind}:${node.name}`);
+        byFile.set(node.filePath, symbols);
+      }
+    }
+
+    const lines: string[] = [];
+    for (const [file, symbols] of byFile) {
+      lines.push(`${file}: ${symbols.join(', ')}`);
+    }
+
+    this.symbolIndexCache = lines.join('\n');
+    return this.symbolIndexCache;
+  }
+
+  /**
+   * Ask Claude CLI to interpret a natural language question into relevant symbol names
+   */
+  private async askClaude(question: string): Promise<string[] | null> {
+    // Check if claude is available (cache result)
+    if (this.claudeAvailable === false) return null;
+
+    const symbolIndex = this.buildSymbolIndex();
+
+    const prompt = `Given the question and codebase symbol index below, identify the single best ENTRY POINT symbol — the one function, component, or route handler where this flow starts.
+
+Rules:
+- Pick ONE symbol that is the starting point a user or request would hit first
+- Prefer page components, route handlers, or top-level functions
+- Do NOT pick utility functions, helpers, or middleware
+
+Return ONLY this JSON, nothing else:
+{"entry": "symbolName"}
+
+Question: "${question}"
+
+Symbol index:
+${symbolIndex}`;
+
+    return new Promise((resolve) => {
+      const timeout = setTimeout(() => {
+        resolve(null);
+      }, 30000);
+
+      execFile('claude', ['-p', prompt, '--output-format', 'text'], {
+        timeout: 30000,
+        maxBuffer: 1024 * 1024,
+      }, (err, stdout) => {
+        clearTimeout(timeout);
+
+        if (err) {
+          this.claudeAvailable = false;
+          resolve(null);
+          return;
+        }
+
+        this.claudeAvailable = true;
+
+        // Parse Claude's response — try object format first, then array fallback
+        try {
+          const text = stdout.trim();
+          // Try to extract JSON object {"entry": ..., "flow": [...]}
+          const objMatch = text.match(/\{[\s\S]*\}/);
+          if (objMatch) {
+            const parsed = JSON.parse(objMatch[0]) as { entry?: string; flow?: string[] };
+            if (parsed.flow && Array.isArray(parsed.flow) && parsed.flow.length > 0) {
+              // Return flow with entry first
+              const names = parsed.flow.map(String);
+              if (parsed.entry && !names.includes(parsed.entry)) {
+                names.unshift(String(parsed.entry));
+              }
+              resolve(names);
+              return;
+            }
+          }
+          // Fallback: try JSON array
+          const arrMatch = text.match(/\[[\s\S]*\]/);
+          if (arrMatch) {
+            const names = JSON.parse(arrMatch[0]) as string[];
+            if (Array.isArray(names) && names.length > 0) {
+              resolve(names.map(String));
+              return;
+            }
+          }
+        } catch {
+          // Parse failed
+        }
+
+        resolve(null);
+      });
+    });
+  }
+
+  /**
+   * Start the visualizer server
+   */
+  async start(options: VisualizerOptions = {}): Promise<{ port: number; url: string }> {
+    const host = options.host || '127.0.0.1';
+    const port = options.port || 0;
+
+    this.server = http.createServer((req, res) => {
+      this.handleRequest(req, res).catch((err) => {
+        console.error('[Visualizer] Request error:', err);
+        res.writeHead(500, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ error: 'Internal server error' }));
+      });
+    });
+
+    return new Promise((resolve, reject) => {
+      this.server!.listen(port, host, () => {
+        const addr = this.server!.address();
+        if (!addr || typeof addr === 'string') {
+          reject(new Error('Failed to get server address'));
+          return;
+        }
+        const serverUrl = `http://${host}:${addr.port}`;
+        resolve({ port: addr.port, url: serverUrl });
+      });
+
+      this.server!.on('error', reject);
+    });
+  }
+
+  /**
+   * Stop the server
+   */
+  stop(): Promise<void> {
+    return new Promise((resolve) => {
+      if (this.server) {
+        this.server.close(() => resolve());
+      } else {
+        resolve();
+      }
+    });
+  }
+
+  private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    const parsedUrl = url.parse(req.url || '/', true);
+    const pathname = parsedUrl.pathname || '/';
+
+    // CORS headers for local development
+    res.setHeader('Access-Control-Allow-Origin', '*');
+    res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
+    res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+
+    if (req.method === 'OPTIONS') {
+      res.writeHead(204);
+      res.end();
+      return;
+    }
+
+    // API routes
+    if (pathname.startsWith('/api/')) {
+      return this.handleAPI(pathname, parsedUrl.query as Record<string, string>, res);
+    }
+
+    // Static file serving
+    return this.serveStatic(pathname, res);
+  }
+
+  private async handleAPI(
+    pathname: string,
+    query: Record<string, string>,
+    res: http.ServerResponse
+  ): Promise<void> {
+    const json = (data: unknown, status = 200) => {
+      res.writeHead(status, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify(data));
+    };
+
+    try {
+      // GET /api/status
+      if (pathname === '/api/status') {
+        const stats = this.cg.getStats();
+        json({ stats, projectRoot: this.projectRoot, projectName: path.basename(this.projectRoot) });
+        return;
+      }
+
+      // GET /api/embeddings/status
+      if (pathname === '/api/embeddings/status') {
+        const config = this.cg.getConfig();
+        const embeddingStats = this.cg.getEmbeddingStats();
+        const isEnabled = config.enableEmbeddings === true;
+        const isInitialized = this.cg.isEmbeddingsInitialized();
+        const totalVectors = embeddingStats?.totalVectors ?? 0;
+        const stats = this.cg.getStats();
+        // Consider ready if we have vectors for at least half the eligible nodes
+        const eligibleNodes = stats.nodeCount - (stats.nodesByKind.file ?? 0) - (stats.nodesByKind.import ?? 0);
+        const isReady = isEnabled && totalVectors > 0 && totalVectors >= eligibleNodes * 0.5;
+        json({ isEnabled, isInitialized, isReady, totalVectors, eligibleNodes });
+        return;
+      }
+
+      // GET /api/embeddings/generate — SSE stream that enables, initializes, and generates embeddings
+      if (pathname === '/api/embeddings/generate') {
+        res.writeHead(200, {
+          'Content-Type': 'text/event-stream',
+          'Cache-Control': 'no-cache',
+          'Connection': 'keep-alive',
+        });
+
+        const send = (event: string, data: unknown) => {
+          res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+        };
+
+        try {
+          // Step 1: Enable embeddings in config
+          send('status', { phase: 'config', message: 'Enabling embeddings...' });
+          const config = this.cg.getConfig();
+          if (!config.enableEmbeddings) {
+            this.cg.updateConfig({ enableEmbeddings: true });
+          }
+
+          // Step 2: Initialize embedding model (downloads on first use)
+          send('status', { phase: 'model', message: 'Loading embedding model (first time may download ~30MB)...' });
+          await this.cg.initializeEmbeddings();
+          send('status', { phase: 'model', message: 'Embedding model ready' });
+
+          // Step 3: Generate embeddings with progress
+          send('status', { phase: 'embedding', message: 'Generating embeddings...' });
+          const count = await this.cg.generateEmbeddings((progress) => {
+            send('progress', {
+              current: progress.current,
+              total: progress.total,
+              nodeName: progress.nodeName,
+              percent: progress.total > 0 ? Math.round((progress.current / progress.total) * 100) : 0,
+            });
+          });
+
+          send('complete', { totalEmbedded: count, message: `Generated ${count} embeddings` });
+        } catch (err) {
+          const message = err instanceof Error ? err.message : String(err);
+          send('error', { message });
+        }
+
+        res.end();
+        return;
+      }
+
+      // GET /api/search?q=...&kind=...&limit=...
+      if (pathname === '/api/search') {
+        const q = query.q || '';
+        const kind = query.kind as NodeKind | undefined;
+        const limit = parseInt(query.limit || '30', 10);
+        if (!q) {
+          json({ results: [] });
+          return;
+        }
+        const results = this.cg.searchNodes(q, { kinds: kind ? [kind] : undefined, limit });
+        json({ results });
+        return;
+      }
+
+      // GET /api/explore?q=...
+      // Find the best entry point, then return its call graph
+      if (pathname === '/api/explore') {
+        const q = query.q || '';
+        if (!q) {
+          json({ nodes: [], edges: [], roots: [], entryPoint: null });
+          return;
+        }
+
+        let entryNodeId: string | null = null;
+        let usedClaude = false;
+        const validKinds: NodeKind[] = ['function', 'method', 'class', 'interface', 'component', 'route'];
+
+        // Try Claude CLI to find the best entry point
+        const claudeNames = await this.askClaude(q);
+        if (claudeNames && claudeNames.length > 0) {
+          usedClaude = true;
+          // Find the entry point in the graph
+          for (const name of claudeNames) {
+            if (entryNodeId) break;
+            const results = this.cg.searchNodes(name, { kinds: validKinds, limit: 3 });
+            for (const r of results) {
+              if (r.node.name.toLowerCase() === name.toLowerCase() ||
+                  r.node.name.toLowerCase().includes(name.toLowerCase()) ||
+                  name.toLowerCase().includes(r.node.name.toLowerCase())) {
+                entryNodeId = r.node.id;
+                break;
+              }
+            }
+          }
+        }
+
+        // Keyword fallback: find best match from query keywords
+        if (!entryNodeId) {
+          const stopWords = new Set(['how', 'does', 'what', 'the', 'is', 'a', 'an', 'and', 'or', 'in', 'to', 'for', 'of', 'with', 'when', 'do', 'it', 'my', 'work', 'works', 'about', 'show', 'me']);
+          const keywords = q.toLowerCase().split(/\s+/)
+            .map(w => w.replace(/[^a-z0-9]/g, ''))
+            .filter(w => w.length >= 2 && !stopWords.has(w));
+
+          for (const kw of keywords) {
+            if (entryNodeId) break;
+            const results = this.cg.searchNodes(kw, { kinds: validKinds, limit: 5 });
+            if (results.length > 0) {
+              entryNodeId = results[0]!.node.id;
+            }
+          }
+        }
+
+        if (!entryNodeId) {
+          json({ nodes: [], edges: [], roots: [], entryPoint: null });
+          return;
+        }
+
+        // Get the call graph from this entry point (depth 3)
+        const callGraph = this.cg.getCallGraph(entryNodeId, 3);
+        const result = serializeSubgraph(callGraph);
+
+        json({
+          nodes: result.nodes,
+          edges: result.edges,
+          roots: [entryNodeId],
+          entryPoint: entryNodeId,
+          usedClaude,
+        });
+        return;
+      }
+
+      // GET /api/overview?limit=...
+      if (pathname === '/api/overview') {
+        const limit = parseInt(query.limit || '50', 10);
+        // Get top-level exported classes, functions, components
+        const kinds: NodeKind[] = ['class', 'function', 'interface', 'component', 'enum', 'type_alias'];
+        const nodes: Node[] = [];
+        for (const kind of kinds) {
+          const kindNodes = this.cg.getNodesByKind(kind);
+          for (const n of kindNodes) {
+            if (n.isExported || n.kind === 'class' || n.kind === 'component') {
+              nodes.push(n);
+            }
+            if (nodes.length >= limit) break;
+          }
+          if (nodes.length >= limit) break;
+        }
+        json({ nodes });
+        return;
+      }
+
+      // GET /api/files
+      if (pathname === '/api/files') {
+        const files = this.cg.getFiles();
+        json({ files });
+        return;
+      }
+
+      // Routes with node ID: /api/node/<id>/...
+      const nodeMatch = pathname.match(/^\/api\/node\/([^/]+)(\/.*)?$/);
+      if (nodeMatch) {
+        const nodeId = decodeURIComponent(nodeMatch[1]!);
+        const sub = nodeMatch[2] || '';
+
+        // GET /api/node/<id>
+        if (!sub || sub === '/') {
+          const node = this.cg.getNode(nodeId);
+          if (!node) {
+            json({ error: 'Node not found' }, 404);
+            return;
+          }
+          const code = await this.cg.getCode(nodeId);
+          const ancestors = this.cg.getAncestors(nodeId);
+          json({ node, code, ancestors });
+          return;
+        }
+
+        // GET /api/node/<id>/callers?depth=...
+        if (sub === '/callers') {
+          const depth = parseInt(query.depth || '1', 10);
+          const items = this.cg.getCallers(nodeId, depth);
+          json({ items });
+          return;
+        }
+
+        // GET /api/node/<id>/callees?depth=...
+        if (sub === '/callees') {
+          const depth = parseInt(query.depth || '1', 10);
+          const items = this.cg.getCallees(nodeId, depth);
+          json({ items });
+          return;
+        }
+
+        // GET /api/node/<id>/children
+        if (sub === '/children') {
+          const children = this.cg.getChildren(nodeId);
+          json({ children });
+          return;
+        }
+
+        // GET /api/node/<id>/impact?depth=...
+        if (sub === '/impact') {
+          const depth = parseInt(query.depth || '2', 10);
+          const subgraph = this.cg.getImpactRadius(nodeId, depth);
+          json(serializeSubgraph(subgraph));
+          return;
+        }
+
+        // GET /api/node/<id>/callgraph?depth=...
+        if (sub === '/callgraph') {
+          const depth = parseInt(query.depth || '2', 10);
+          const subgraph = this.cg.getCallGraph(nodeId, depth);
+          json(serializeSubgraph(subgraph));
+          return;
+        }
+
+        // GET /api/node/<id>/context
+        if (sub === '/context') {
+          const context = this.cg.getContext(nodeId);
+          json({ context });
+          return;
+        }
+
+        json({ error: 'Unknown endpoint' }, 404);
+        return;
+      }
+
+      // GET /api/file-nodes?path=...
+      if (pathname === '/api/file-nodes') {
+        const filePath = query.path || '';
+        if (!filePath) {
+          json({ error: 'path parameter required' }, 400);
+          return;
+        }
+        const nodes = this.cg.getNodesInFile(filePath);
+        json({ nodes });
+        return;
+      }
+
+      json({ error: 'Unknown API endpoint' }, 404);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      json({ error: message }, 500);
+    }
+  }
+
+  private serveStatic(pathname: string, res: http.ServerResponse): void {
+    if (pathname === '/' || pathname === '/index.html') {
+      pathname = '/index.html';
+    }
+
+    // Resolve from the public directory next to this file
+    const publicDir = path.join(__dirname, 'public');
+    const filePath = path.join(publicDir, pathname);
+
+    // Security: prevent directory traversal
+    if (!filePath.startsWith(publicDir)) {
+      res.writeHead(403);
+      res.end('Forbidden');
+      return;
+    }
+
+    const ext = path.extname(filePath).toLowerCase();
+    const mimeTypes: Record<string, string> = {
+      '.html': 'text/html',
+      '.css': 'text/css',
+      '.js': 'application/javascript',
+      '.json': 'application/json',
+      '.png': 'image/png',
+      '.svg': 'image/svg+xml',
+      '.ico': 'image/x-icon',
+    };
+
+    try {
+      const content = fs.readFileSync(filePath);
+      res.writeHead(200, { 'Content-Type': mimeTypes[ext] || 'application/octet-stream' });
+      res.end(content);
+    } catch {
+      res.writeHead(404, { 'Content-Type': 'text/plain' });
+      res.end('Not found');
+    }
+  }
+}