diff --git a/README.md b/README.md index 1d2a11b..f8f39e9 100644 --- a/README.md +++ b/README.md @@ -439,6 +439,30 @@ The `.codegraph/config.json` file controls indexing: **Indexing is slow** — Check that `node_modules` and other large directories are excluded. Use `--quiet` to reduce output overhead. +**Indexing is slow / MCP `database is locked` / WASM fallback active** — `codegraph` ships with a WASM SQLite fallback for environments where `better-sqlite3` (a native module, declared as `optionalDependencies`) can't install. The fallback is 5-10x slower than the native backend and uses a journal mode that lets writers block readers, so MCP queries can also hit `database is locked` while indexing runs. Run `codegraph status` and look at the `Backend:` line: + +- `Backend: native` — you're on the fast path, nothing to do. +- `Backend: wasm` — you're on the slow fallback. Common causes: missing C build tools, prebuilt binary unavailable for your Node version, or your Node version changed after install. Fix: + + ```bash + # macOS + xcode-select --install # installs the C compiler + + # Linux (Debian / Ubuntu) + sudo apt install build-essential python3 make + + # Linux (RHEL / Fedora) + sudo yum groupinstall "Development Tools" + + # Then rebuild on any platform: + npm rebuild better-sqlite3 + + # Or force-include as a hard dep: + npm install better-sqlite3 --save + ``` + + After the fix, `codegraph status` should show `Backend: native`. + **MCP server not connecting** — Ensure the project is initialized/indexed, verify the path in your MCP config, and check that `codegraph serve --mcp` works from the command line. **Missing symbols** — The MCP server auto-syncs on save (wait a couple seconds). Run `codegraph sync` manually if needed. Check that the file's language is supported and isn't excluded by config patterns. diff --git a/__tests__/sqlite-backend.test.ts b/__tests__/sqlite-backend.test.ts new file mode 100644 index 0000000..9fe139c --- /dev/null +++ b/__tests__/sqlite-backend.test.ts @@ -0,0 +1,86 @@ +/** + * SQLite backend visibility tests + * + * Pins the WASM-fallback banner content + the per-instance backend + * tracking. Closes the visibility gap behind issue #138. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { + buildWasmFallbackBanner, + WASM_FALLBACK_FIX_RECIPE, +} from '../src/db/sqlite-adapter'; +import { DatabaseConnection } from '../src/db'; +import { CodeGraph } from '../src'; + +describe('buildWasmFallbackBanner — fix-recipe content', () => { + it('includes the macOS / Linux / cross-platform fix commands', () => { + const banner = buildWasmFallbackBanner(); + expect(banner).toContain('WASM SQLite fallback active'); + expect(banner).toContain('5-10x slower'); + expect(banner).toContain('xcode-select --install'); + expect(banner).toContain('apt install build-essential'); + expect(banner).toContain('npm rebuild better-sqlite3'); + expect(banner).toContain('npm install better-sqlite3 --save'); + expect(banner).toContain('codegraph status'); + }); + + it('appends the native load error when one is provided', () => { + const banner = buildWasmFallbackBanner( + "Cannot find module 'better-sqlite3'" + ); + expect(banner).toContain( + "Native load error: Cannot find module 'better-sqlite3'" + ); + }); + + it('omits the load-error block when no error is supplied', () => { + const banner = buildWasmFallbackBanner(); + expect(banner).not.toContain('Native load error:'); + }); +}); + +describe('WASM_FALLBACK_FIX_RECIPE — single source of truth', () => { + it('mentions the three recovery commands', () => { + expect(WASM_FALLBACK_FIX_RECIPE).toContain('xcode-select --install'); + expect(WASM_FALLBACK_FIX_RECIPE).toContain('npm rebuild better-sqlite3'); + expect(WASM_FALLBACK_FIX_RECIPE).toContain( + 'npm install better-sqlite3 --save' + ); + }); +}); + +describe('DatabaseConnection — per-instance backend reporting', () => { + let dir: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-backend-')); + }); + + afterEach(() => { + if (fs.existsSync(dir)) { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('reports a concrete backend (native or wasm) for an initialized DB', () => { + const dbPath = path.join(dir, 'test.db'); + const conn = DatabaseConnection.initialize(dbPath); + const backend = conn.getBackend(); + expect(['native', 'wasm']).toContain(backend); + conn.close(); + }); + + it('CodeGraph.getBackend() delegates to the underlying DatabaseConnection', async () => { + fs.writeFileSync(path.join(dir, 'x.ts'), `export function x(): void {}\n`); + const cg = await CodeGraph.init(dir, { index: true }); + try { + expect(['native', 'wasm']).toContain(cg.getBackend()); + } finally { + cg.destroy(); + } + }); +}); diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index 2843e67..f29b846 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -643,6 +643,7 @@ program const cg = await CodeGraph.open(projectPath); const stats = cg.getStats(); const changes = cg.getChangedFiles(); + const backend = cg.getBackend(); // JSON output mode if (options.json) { @@ -653,6 +654,7 @@ program nodeCount: stats.nodeCount, edgeCount: stats.edgeCount, dbSizeBytes: stats.dbSizeBytes, + backend, nodesByKind: stats.nodesByKind, languages: Object.entries(stats.filesByLanguage).filter(([, count]) => count > 0).map(([lang]) => lang), pendingChanges: { @@ -677,6 +679,14 @@ program console.log(` Nodes: ${formatNumber(stats.nodeCount)}`); console.log(` Edges: ${formatNumber(stats.edgeCount)}`); console.log(` DB Size: ${(stats.dbSizeBytes / 1024 / 1024).toFixed(2)} MB`); + // Surface the active SQLite backend so users can spot the silent + // WASM fallback (5-10x slower). better-sqlite3 is in + // `optionalDependencies`, so `npm install` succeeds without it + // when the native build fails. + const backendLabel = backend === 'native' + ? chalk.green('native') + : chalk.yellow('wasm — slower fallback; run `npm rebuild better-sqlite3`'); + console.log(` Backend: ${backendLabel}`); console.log(); // Node breakdown diff --git a/src/db/index.ts b/src/db/index.ts index 34e9933..3e49074 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -4,13 +4,13 @@ * Handles SQLite database initialization and connection management. */ -import { SqliteDatabase, createDatabase } from './sqlite-adapter'; +import { SqliteDatabase, SqliteBackend, createDatabase } from './sqlite-adapter'; import * as fs from 'fs'; import * as path from 'path'; import { SchemaVersion } from '../types'; import { runMigrations, getCurrentVersion, CURRENT_SCHEMA_VERSION } from './migrations'; -export { SqliteDatabase, getActiveBackend } from './sqlite-adapter'; +export { SqliteDatabase, SqliteBackend, WASM_FALLBACK_FIX_RECIPE } from './sqlite-adapter'; /** * Database connection wrapper with lifecycle management @@ -18,10 +18,12 @@ export { SqliteDatabase, getActiveBackend } from './sqlite-adapter'; export class DatabaseConnection { private db: SqliteDatabase; private dbPath: string; + private backend: SqliteBackend; - private constructor(db: SqliteDatabase, dbPath: string) { + private constructor(db: SqliteDatabase, dbPath: string, backend: SqliteBackend) { this.db = db; this.dbPath = dbPath; + this.backend = backend; } /** @@ -35,7 +37,7 @@ export class DatabaseConnection { } // Create and configure database - const db = createDatabase(dbPath); + const { db, backend } = createDatabase(dbPath); // Enable foreign keys and WAL mode for better performance db.pragma('foreign_keys = ON'); @@ -62,7 +64,7 @@ export class DatabaseConnection { ).run(CURRENT_SCHEMA_VERSION, Date.now(), 'Initial schema includes all migrations'); } - return new DatabaseConnection(db, dbPath); + return new DatabaseConnection(db, dbPath, backend); } /** @@ -73,7 +75,7 @@ export class DatabaseConnection { throw new Error(`Database not found: ${dbPath}`); } - const db = createDatabase(dbPath); + const { db, backend } = createDatabase(dbPath); // Enable foreign keys and WAL mode db.pragma('foreign_keys = ON'); @@ -88,7 +90,7 @@ export class DatabaseConnection { db.pragma('mmap_size = 268435456'); // Check and run migrations if needed - const conn = new DatabaseConnection(db, dbPath); + const conn = new DatabaseConnection(db, dbPath, backend); const currentVersion = getCurrentVersion(db); if (currentVersion < CURRENT_SCHEMA_VERSION) { @@ -105,6 +107,15 @@ export class DatabaseConnection { return this.db; } + /** + * Get the SQLite backend serving this connection. Per-instance so + * MCP cross-project queries report the right backend even when + * multiple project DBs are open in the same process. + */ + getBackend(): SqliteBackend { + return this.backend; + } + /** * Get database file path */ diff --git a/src/db/sqlite-adapter.ts b/src/db/sqlite-adapter.ts index 01155fc..c3d31c8 100644 --- a/src/db/sqlite-adapter.ts +++ b/src/db/sqlite-adapter.ts @@ -22,13 +22,51 @@ export interface SqliteDatabase { export type SqliteBackend = 'native' | 'wasm'; -let activeBackend: SqliteBackend | null = null; +/** + * One-line summary of the recovery steps shown when WASM fallback is + * active. Single source of truth so the recipe can't drift between the + * stderr banner and the MCP status formatter. + */ +export const WASM_FALLBACK_FIX_RECIPE = + '`xcode-select --install` (macOS) or `apt install build-essential` (Debian/Ubuntu), ' + + 'then `npm rebuild better-sqlite3`, or `npm install better-sqlite3 --save` to force-include it.'; /** - * Get the currently active SQLite backend. + * Multi-line banner shown to stderr when `createDatabase` falls back to + * WASM. Replaces a one-line `console.warn` that MCP transports (which + * take stdout for the protocol) typically swallow, leaving users on a + * 5-10x slower backend with no signal. + * + * Exported for unit testing — pinning the recipe content prevents + * future edits from silently stripping the recovery commands. */ -export function getActiveBackend(): SqliteBackend | null { - return activeBackend; +export function buildWasmFallbackBanner(nativeError?: string): string { + const sep = '─'.repeat(72); + const lines = [ + sep, + '[CodeGraph] WASM SQLite fallback active (better-sqlite3 unavailable)', + sep, + 'Indexing and sync will be 5-10x slower than the native backend.', + '', + 'Fix on macOS:', + ' xcode-select --install # install C build tools', + ' npm rebuild better-sqlite3 # rebuild native binding for current Node', + '', + 'Fix on Linux:', + ' sudo apt install build-essential python3 make # Debian/Ubuntu', + ' # or: sudo yum groupinstall "Development Tools" # RHEL/Fedora', + ' npm rebuild better-sqlite3', + '', + 'Or force-include as a hard dependency on any platform:', + ' npm install better-sqlite3 --save', + '', + 'Verify after fix: `codegraph status` should show `Backend: native`.', + ]; + if (nativeError) { + lines.push('', `Native load error: ${nativeError}`); + } + lines.push(sep); + return lines.join('\n'); } /** @@ -192,9 +230,13 @@ class WasmDatabaseAdapter implements SqliteDatabase { /** * Create a database connection. Tries native better-sqlite3 first, - * falls back to node-sqlite3-wasm. + * falls back to node-sqlite3-wasm. Returns the active backend + * alongside the db so each `DatabaseConnection` can report its own + * backend per-instance — MCP can open multiple project DBs in one + * process (`tools.ts` getCodeGraph cache), so a process-global would + * race / overwrite. */ -export function createDatabase(dbPath: string): SqliteDatabase { +export function createDatabase(dbPath: string): { db: SqliteDatabase; backend: SqliteBackend } { let nativeError: string | undefined; let wasmError: string | undefined; @@ -203,8 +245,7 @@ export function createDatabase(dbPath: string): SqliteDatabase { // eslint-disable-next-line @typescript-eslint/no-require-imports const Database = require('better-sqlite3'); const db = new Database(dbPath); - activeBackend = 'native'; - return db as SqliteDatabase; + return { db: db as SqliteDatabase, backend: 'native' }; } catch (error) { nativeError = error instanceof Error ? error.message : String(error); } @@ -212,9 +253,8 @@ export function createDatabase(dbPath: string): SqliteDatabase { // Fall back to WASM try { const db = new WasmDatabaseAdapter(dbPath); - activeBackend = 'wasm'; - console.warn('[CodeGraph] Using WASM SQLite backend (native better-sqlite3 unavailable)'); - return db; + console.warn(buildWasmFallbackBanner(nativeError)); + return { db, backend: 'wasm' }; } catch (error) { wasmError = error instanceof Error ? error.message : String(error); } diff --git a/src/index.ts b/src/index.ts index 0ff1e09..7d58674 100644 --- a/src/index.ts +++ b/src/index.ts @@ -612,6 +612,16 @@ export class CodeGraph { return stats; } + /** + * Active SQLite backend for this project's connection. `wasm` means + * the native better-sqlite3 install failed and the WASM fallback is + * serving requests at 5-10x the latency. Surfaced via `codegraph + * status` and the `codegraph_status` MCP tool. + */ + getBackend(): import('./db').SqliteBackend { + return this.db.getBackend(); + } + // =========================================================================== // Node Operations // =========================================================================== diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 5371314..e796cfc 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -11,6 +11,7 @@ import { writeFileSync, readFileSync, existsSync } from 'fs'; import { clamp, validatePathWithinRoot } from '../utils'; import { tmpdir } from 'os'; import { join } from 'path'; +import { WASM_FALLBACK_FIX_RECIPE } from '../db'; /** Maximum output length to prevent context bloat (characters) */ const MAX_OUTPUT_LENGTH = 15000; @@ -973,10 +974,23 @@ export class ToolHandler { `**Total nodes:** ${stats.nodeCount}`, `**Total edges:** ${stats.edgeCount}`, `**Database size:** ${(stats.dbSizeBytes / 1024 / 1024).toFixed(2)} MB`, - '', - '### Nodes by Kind:', ]; + // Surface the active SQLite backend. Without this, users on the + // silent WASM fallback (better-sqlite3 install failed) see "slow" + // indexing and DB-lock errors with no signal of why. + const backend = cg.getBackend(); + if (backend === 'native') { + lines.push(`**Backend:** native (better-sqlite3)`); + } else { + lines.push( + `**Backend:** ⚠ wasm (better-sqlite3 unavailable) — ` + + `5-10x slower than native. Fix: ${WASM_FALLBACK_FIX_RECIPE}` + ); + } + + lines.push('', '### Nodes by Kind:'); + for (const [kind, count] of Object.entries(stats.nodesByKind)) { if ((count as number) > 0) { lines.push(`- ${kind}: ${count}`);