From 0fd259b5547711b9b8c24748fb65eeaa4dfa88cf Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Tue, 8 Sep 2026 19:57:48 -0500 Subject: [PATCH] fix(db): graceful FTS5 fallback when Node.js build lacks FTS5 (#1532) (#1810) * fix(db): graceful FTS5 fallback when Node.js build lacks FTS5 support (#1532) Official Node.js binaries do not compile FTS5 by default, causing codegraph init to fail with 'no such module: fts5'. Added runtime FTS5 detection: - Split schema execution to try FTS5 separately, skip on failure with warning - Added fts5Available flag to DatabaseConnection and QueryBuilder - Bulk-load and search paths skip FTS5 operations when unavailable - Search falls back to LIKE + fuzzy matching when FTS5 is missing (cherry picked from commit ed708b7f60540367d8a810b0388aaa05ce0a0933) * fix(db): preserve core schema during FTS5 fallback (#1532) Keep required tables and indexes after the FTS triggers outside the optional schema block in the upstream #1625 fix. Without this boundary, simulated-missing-FTS5 indexing still fails on name_segment_vocab. Add seven regressions using real SQLite with FTS5 creation intercepted, covering initialization/open, LIKE and fuzzy search, non-FTS schema parity, bulk no-ops, and real FTS5 search and bulk-load recovery. Credit @aniruddhaadak80 under Unreleased fixes. Validation on Linux x64 with Node v22.19.0: - npm run build passed, including viewer and grammar asset checks. - 34 tests passed across fts5-fallback, node-sqlite-backend, sqlite-backend, and db-perf. - Rebuilt CodeGraph initialization, indexing, reopening, search, and cross-file callers passed with simulated missing FTS5 and real FTS5. Fixes #1532. Supersedes #1625. --------- Co-authored-by: Aniruddha Adak Co-authored-by: Colby McHenry --- CHANGELOG.md | 2 + __tests__/fts5-fallback.test.ts | 166 ++++++++++++++++++++++++++++++++ src/db/index.ts | 59 +++++++++++- src/db/queries.ts | 14 ++- 4 files changed, 234 insertions(+), 7 deletions(-) create mode 100644 __tests__/fts5-fallback.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ffe37a0..89a4d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,6 +140,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). #### MCP / indexing +- Indexing now succeeds when Node.js's SQLite lacks FTS5, with search falling back to name and fuzzy matching; thanks @aniruddhaadak80. (#1532) + - `codegraph_explore` now makes clear that suggested call counts are advisory, so agents keep exploring when an answer is incomplete; thanks @rongbc. (#1504, #1570) - C++ functions following anonymous namespaces containing raw-string templates are now indexed correctly, even when template text resembles an unfinished macro call. (#1505) diff --git a/__tests__/fts5-fallback.test.ts b/__tests__/fts5-fallback.test.ts new file mode 100644 index 0000000..19d6022 --- /dev/null +++ b/__tests__/fts5-fallback.test.ts @@ -0,0 +1,166 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { DatabaseConnection } from '../src/db'; +import { QueryBuilder } from '../src/db/queries'; +import { Node } from '../src/types'; + +// Use real SQLite for every operation except the unsupported-module error. +// This must exercise fallback even when the test runner's Node has FTS5. +const { DatabaseSync } = require('node:sqlite'); + +function simulateMissingFts5(): () => number { + const exec = DatabaseSync.prototype.exec; + let attempts = 0; + vi.spyOn(DatabaseSync.prototype, 'exec').mockImplementation(function (this: unknown, sql: string) { + if (/CREATE VIRTUAL TABLE\b[^;]*\bUSING fts5\s*\(/i.test(sql)) { + attempts++; + throw new Error('no such module: fts5'); + } + return exec.call(this, sql); + }); + return () => attempts; +} + +function makeNode(name: string, docstring?: string): Node { + return { + id: name, + kind: 'function', + name, + qualifiedName: name, + filePath: 'src/users.ts', + language: 'typescript', + startLine: 1, + endLine: 1, + startColumn: 0, + endColumn: 0, + docstring, + updatedAt: Date.now(), + }; +} + +describe('FTS5 fallback (#1532)', () => { + let dir: string; + let connections: DatabaseConnection[]; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-fts5-fallback-')); + connections = []; + }); + + afterEach(() => { + vi.restoreAllMocks(); + for (const connection of connections) connection.close(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + function initialize(filename = 'test.db'): DatabaseConnection { + const connection = DatabaseConnection.initialize(path.join(dir, filename)); + connections.push(connection); + return connection; + } + + function reopen(connection: DatabaseConnection): DatabaseConnection { + connection.close(); + const reopened = DatabaseConnection.open(path.join(dir, 'test.db')); + connections.push(reopened); + return reopened; + } + + it.each(['initialization', 'reopening'])('uses LIKE and fuzzy search after %s without FTS5', (state) => { + const attempts = simulateMissingFts5(); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + let connection = initialize(); + + expect(attempts()).toBe(1); + expect(connection.fts5Available).toBe(false); + expect(warn).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('no such module: fts5')); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('LIKE + fuzzy matching')); + + if (state === 'reopening') connection = reopen(connection); + expect(connection.fts5Available).toBe(false); + + const db = connection.getDb(); + expect(db.prepare("SELECT name FROM sqlite_master WHERE name = 'nodes_fts' OR name IN ('nodes_ai', 'nodes_ad', 'nodes_au')").all()).toEqual([]); + const exec = vi.spyOn(db, 'exec'); + connection.beginBulkNodeLoad(); + connection.endBulkNodeLoad(); + expect(exec).not.toHaveBeenCalled(); + + const queries = new QueryBuilder(db); + queries.insertNodes([makeNode('getUser'), makeNode('getUserProfile')]); + const prepare = vi.spyOn(db, 'prepare'); + + expect(queries.searchNodes('User').map(result => result.node.name)).toEqual(expect.arrayContaining(['getUser', 'getUserProfile'])); + expect(queries.searchNodes('getUssr').map(result => result.node.name)).toEqual(['getUser']); + // A failed MATCH query is already caught by searchNodesFTS; pin that the + // unavailable path skips the FTS query entirely, rather than retrying it. + expect(prepare.mock.calls.some(([sql]) => /\bnodes_fts\b/.test(sql))).toBe(false); + + queries.setMetadata('project_name', 'fts5-fallback'); + expect(queries.getMetadata('project_name')).toBe('fts5-fallback'); + }); + + it('keeps every non-FTS table and index when FTS5 creation fails', () => { + const control = initialize('control.db'); + const nonFtsSchema = (connection: DatabaseConnection) => connection.getDb().prepare(` + SELECT type, name, sql FROM sqlite_master + WHERE name NOT LIKE 'nodes_fts%' + AND name NOT IN ('nodes_ai', 'nodes_ad', 'nodes_au') + ORDER BY type, name + `).all(); + const expected = nonFtsSchema(control); + + simulateMissingFts5(); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const fallback = initialize(); + + expect(fallback.fts5Available).toBe(false); + expect(nonFtsSchema(fallback)).toEqual(expected); + }); + + it.each(['initialization', 'reopening'])('uses real FTS5 after %s', (state) => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + let connection = initialize(); + expect(connection.fts5Available).toBe(true); + new QueryBuilder(connection.getDb()).insertNode(makeNode('loadRecord', 'quasar nebula')); + + if (state === 'reopening') connection = reopen(connection); + expect(connection.fts5Available).toBe(true); + const queries = new QueryBuilder(connection.getDb()); + // Only the docstring contains this token: LIKE/fuzzy name search cannot + // make this assertion pass if the FTS path is accidentally disabled. + expect(queries.searchNodes('nebula').map(result => result.node.name)).toEqual(['loadRecord']); + expect(warn).not.toHaveBeenCalled(); + }); + + it('rebuilds real FTS5 after a bulk node load', () => { + const connection = initialize(); + const queries = new QueryBuilder(connection.getDb()); + + connection.beginBulkNodeLoad(); + queries.insertNode(makeNode('loadRecord', 'quasar nebula')); + expect(queries.searchNodes('nebula')).toEqual([]); + connection.endBulkNodeLoad(); + + expect(queries.searchNodes('nebula').map(result => result.node.name)).toEqual(['loadRecord']); + queries.insertNode(makeNode('saveRecord', 'pulsar supernova')); + expect(queries.searchNodes('supernova').map(result => result.node.name)).toEqual(['saveRecord']); + }); + + it('repairs an interrupted real FTS5 bulk load on open', () => { + let connection = initialize(); + connection.beginBulkNodeLoad(); + new QueryBuilder(connection.getDb()).insertNode(makeNode('loadRecord', 'quasar nebula')); + + connection = reopen(connection); + + expect(connection.fts5Available).toBe(true); + const queries = new QueryBuilder(connection.getDb()); + expect(queries.searchNodes('nebula').map(result => result.node.name)).toEqual(['loadRecord']); + queries.insertNode(makeNode('saveRecord', 'pulsar supernova')); + expect(queries.searchNodes('supernova').map(result => result.node.name)).toEqual(['saveRecord']); + }); +}); diff --git a/src/db/index.ts b/src/db/index.ts index f01d195..c7ec1ee 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -83,10 +83,17 @@ export class DatabaseConnection { */ private openedInode: string | null; - private constructor(db: SqliteDatabase, dbPath: string, backend: SqliteBackend) { + /** + * Whether FTS5 is available in this Node.js build. When false, search + * falls back to LIKE + fuzzy matching (#1532). + */ + readonly fts5Available: boolean; + + private constructor(db: SqliteDatabase, dbPath: string, backend: SqliteBackend, fts5Available: boolean) { this.db = db; this.dbPath = dbPath; this.backend = backend; + this.fts5Available = fts5Available; this.openedInode = statInode(dbPath); } @@ -105,10 +112,41 @@ export class DatabaseConnection { configureConnection(db); - // Run schema initialization + // Run schema initialization, splitting FTS5 from the rest so + // codegraph still works when Node.js was built without FTS5 (#1532). const schemaPath = path.join(__dirname, 'schema.sql'); const schema = fs.readFileSync(schemaPath, 'utf-8'); - db.exec(schema); + + const FTS5_MARKER = '-- Full-text search index on node names, docstrings, and signatures'; + const ftsIdx = schema.indexOf(FTS5_MARKER); + let fts5Available = true; + + if (ftsIdx >= 0) { + const preFts = schema.slice(0, ftsIdx); + // FTS ends after the update trigger; required tables and indexes follow + // it in schema.sql and must still be created when FTS5 is unavailable. + const ftsSection = schema.slice(ftsIdx).match( + /^[\s\S]*?CREATE TRIGGER IF NOT EXISTS nodes_au\b[\s\S]*?END;/ + )?.[0]; + if (!ftsSection) throw new Error('schema.sql: FTS5 update trigger not found'); + // Execute everything before FTS5 first + db.exec(preFts); + // Try FTS5; if it fails, skip it and continue with LIKE-only search + try { + db.exec(ftsSection); + } catch (err: any) { + fts5Available = false; + const msg = err?.message ?? String(err); + console.warn( + `[codegraph] FTS5 not available in this Node.js build (${msg}). ` + + `Search will fall back to LIKE + fuzzy matching. ` + + `For full-text search, use a Node.js build with FTS5 enabled.` + ); + } + db.exec(schema.slice(ftsIdx + ftsSection.length)); + } else { + db.exec(schema); + } // Record current schema version so migrations aren't re-applied on open const currentVersion = getCurrentVersion(db); @@ -118,7 +156,7 @@ export class DatabaseConnection { ).run(CURRENT_SCHEMA_VERSION, Date.now(), 'Initial schema includes all migrations'); } - return new DatabaseConnection(db, dbPath, backend); + return new DatabaseConnection(db, dbPath, backend, fts5Available); } /** @@ -133,8 +171,16 @@ export class DatabaseConnection { configureConnection(db); + // Detect FTS5 availability for search fallback (#1532) + let fts5Available = true; + try { + db.exec("SELECT * FROM nodes_fts LIMIT 0"); + } catch { + fts5Available = false; + } + // Check and run migrations if needed - const conn = new DatabaseConnection(db, dbPath, backend); + const conn = new DatabaseConnection(db, dbPath, backend, fts5Available); const currentVersion = getCurrentVersion(db); if (currentVersion < CURRENT_SCHEMA_VERSION) { @@ -169,6 +215,7 @@ export class DatabaseConnection { * row written by anyone during the window is captured by the rebuild. */ beginBulkNodeLoad(): void { + if (!this.fts5Available) return; for (const t of DatabaseConnection.FTS_TRIGGER_NAMES) { this.db.exec(`DROP TRIGGER IF EXISTS ${t}`); } @@ -181,6 +228,7 @@ export class DatabaseConnection { * IF NOT EXISTS). */ endBulkNodeLoad(): void { + if (!this.fts5Available) return; this.db.exec(`INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')`); this.recreateFtsTriggers(); } @@ -355,6 +403,7 @@ export class DatabaseConnection { /** Recreate the FTS triggers + rebuild if a bulk-load window never closed. */ private healBulkNodeLoad(): void { + if (!this.fts5Available) return; const row = this.db .prepare( `SELECT count(*) AS c FROM sqlite_master WHERE type = 'trigger' AND name IN ('nodes_ai','nodes_ad','nodes_au')` diff --git a/src/db/queries.ts b/src/db/queries.ts index 0d45562..37c0788 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -244,6 +244,9 @@ export class QueryBuilder { private projectNameTokens: Set = new Set(); private isDeprioritizedPath: ((filePath: string) => boolean) | undefined; + // FTS5 availability flag — detected once at construction time (#1532) + private _fts5Available: boolean | undefined; + // Node cache for frequently accessed nodes (LRU-style, max 1000 entries) private nodeCache: Map = new Map(); private readonly maxCacheSize = 1000; @@ -340,6 +343,13 @@ export class QueryBuilder { constructor(db: SqliteDatabase) { this.db = db; + // Detect FTS5 availability once (#1532) + try { + db.prepare("SELECT * FROM nodes_fts LIMIT 0").get(); + this._fts5Available = true; + } catch { + this._fts5Available = false; + } } /** @@ -1300,9 +1310,9 @@ export class QueryBuilder { const kinds = mergedKinds; const languages = mergedLanguages; - // First try FTS5 with prefix matching + // First try FTS5 with prefix matching (skip if FTS5 not available, #1532) let results = text - ? this.searchNodesFTS(text, { kinds, languages, limit, offset }) + ? (this._fts5Available !== false ? this.searchNodesFTS(text, { kinds, languages, limit, offset }) : []) // Over-fetch by 5× when running filter-only (no text). The // post-scoring path: + name: filters can be very selective, so // a smaller multiplier risks returning fewer than `limit`