fix(db): graceful FTS5 fallback when Node.js build lacks FTS5 (#1532) (#1810)

* fix(db): graceful FTS5 fallback when Node.js build lacks FTS5 support (#1532)

Official Node.js binaries do not compile FTS5 by default, causing codegraph
init to fail with 'no such module: fts5'. Added runtime FTS5 detection:
- Split schema execution to try FTS5 separately, skip on failure with warning
- Added fts5Available flag to DatabaseConnection and QueryBuilder
- Bulk-load and search paths skip FTS5 operations when unavailable
- Search falls back to LIKE + fuzzy matching when FTS5 is missing

(cherry picked from commit ed708b7f60540367d8a810b0388aaa05ce0a0933)

* fix(db): preserve core schema during FTS5 fallback (#1532)

Keep required tables and indexes after the FTS triggers outside the
optional schema block in the upstream #1625 fix. Without this boundary,
simulated-missing-FTS5 indexing still fails on name_segment_vocab.

Add seven regressions using real SQLite with FTS5 creation intercepted,
covering initialization/open, LIKE and fuzzy search, non-FTS schema parity,
bulk no-ops, and real FTS5 search and bulk-load recovery. Credit
@aniruddhaadak80 under Unreleased fixes.

Validation on Linux x64 with Node v22.19.0:
- npm run build passed, including viewer and grammar asset checks.
- 34 tests passed across fts5-fallback, node-sqlite-backend,
  sqlite-backend, and db-perf.
- Rebuilt CodeGraph initialization, indexing, reopening, search, and
  cross-file callers passed with simulated missing FTS5 and real FTS5.

Fixes #1532.
Supersedes #1625.

---------

Co-authored-by: Aniruddha Adak <aniruddhaadak80@users.noreply.github.com>
Co-authored-by: Colby McHenry <colbymchenry@users.noreply.github.com>

CHANGELOG.md

@@ -140,6 +140,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 #### MCP / indexing
 
+- Indexing now succeeds when Node.js's SQLite lacks FTS5, with search falling back to name and fuzzy matching; thanks @aniruddhaadak80. (#1532)
+
 - `codegraph_explore` now makes clear that suggested call counts are advisory, so agents keep exploring when an answer is incomplete; thanks @rongbc. (#1504, #1570)
 
 - C++ functions following anonymous namespaces containing raw-string templates are now indexed correctly, even when template text resembles an unfinished macro call. (#1505)

__tests__/fts5-fallback.test.ts

@@ -0,0 +1,166 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { DatabaseConnection } from '../src/db';
+import { QueryBuilder } from '../src/db/queries';
+import { Node } from '../src/types';
+
+// Use real SQLite for every operation except the unsupported-module error.
+// This must exercise fallback even when the test runner's Node has FTS5.
+const { DatabaseSync } = require('node:sqlite');
+
+function simulateMissingFts5(): () => number {
+  const exec = DatabaseSync.prototype.exec;
+  let attempts = 0;
+  vi.spyOn(DatabaseSync.prototype, 'exec').mockImplementation(function (this: unknown, sql: string) {
+    if (/CREATE VIRTUAL TABLE\b[^;]*\bUSING fts5\s*\(/i.test(sql)) {
+      attempts++;
+      throw new Error('no such module: fts5');
+    }
+    return exec.call(this, sql);
+  });
+  return () => attempts;
+}
+
+function makeNode(name: string, docstring?: string): Node {
+  return {
+    id: name,
+    kind: 'function',
+    name,
+    qualifiedName: name,
+    filePath: 'src/users.ts',
+    language: 'typescript',
+    startLine: 1,
+    endLine: 1,
+    startColumn: 0,
+    endColumn: 0,
+    docstring,
+    updatedAt: Date.now(),
+  };
+}
+
+describe('FTS5 fallback (#1532)', () => {
+  let dir: string;
+  let connections: DatabaseConnection[];
+
+  beforeEach(() => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-fts5-fallback-'));
+    connections = [];
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    for (const connection of connections) connection.close();
+    fs.rmSync(dir, { recursive: true, force: true });
+  });
+
+  function initialize(filename = 'test.db'): DatabaseConnection {
+    const connection = DatabaseConnection.initialize(path.join(dir, filename));
+    connections.push(connection);
+    return connection;
+  }
+
+  function reopen(connection: DatabaseConnection): DatabaseConnection {
+    connection.close();
+    const reopened = DatabaseConnection.open(path.join(dir, 'test.db'));
+    connections.push(reopened);
+    return reopened;
+  }
+
+  it.each(['initialization', 'reopening'])('uses LIKE and fuzzy search after %s without FTS5', (state) => {
+    const attempts = simulateMissingFts5();
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+    let connection = initialize();
+
+    expect(attempts()).toBe(1);
+    expect(connection.fts5Available).toBe(false);
+    expect(warn).toHaveBeenCalledOnce();
+    expect(warn).toHaveBeenCalledWith(expect.stringContaining('no such module: fts5'));
+    expect(warn).toHaveBeenCalledWith(expect.stringContaining('LIKE + fuzzy matching'));
+
+    if (state === 'reopening') connection = reopen(connection);
+    expect(connection.fts5Available).toBe(false);
+
+    const db = connection.getDb();
+    expect(db.prepare("SELECT name FROM sqlite_master WHERE name = 'nodes_fts' OR name IN ('nodes_ai', 'nodes_ad', 'nodes_au')").all()).toEqual([]);
+    const exec = vi.spyOn(db, 'exec');
+    connection.beginBulkNodeLoad();
+    connection.endBulkNodeLoad();
+    expect(exec).not.toHaveBeenCalled();
+
+    const queries = new QueryBuilder(db);
+    queries.insertNodes([makeNode('getUser'), makeNode('getUserProfile')]);
+    const prepare = vi.spyOn(db, 'prepare');
+
+    expect(queries.searchNodes('User').map(result => result.node.name)).toEqual(expect.arrayContaining(['getUser', 'getUserProfile']));
+    expect(queries.searchNodes('getUssr').map(result => result.node.name)).toEqual(['getUser']);
+    // A failed MATCH query is already caught by searchNodesFTS; pin that the
+    // unavailable path skips the FTS query entirely, rather than retrying it.
+    expect(prepare.mock.calls.some(([sql]) => /\bnodes_fts\b/.test(sql))).toBe(false);
+
+    queries.setMetadata('project_name', 'fts5-fallback');
+    expect(queries.getMetadata('project_name')).toBe('fts5-fallback');
+  });
+
+  it('keeps every non-FTS table and index when FTS5 creation fails', () => {
+    const control = initialize('control.db');
+    const nonFtsSchema = (connection: DatabaseConnection) => connection.getDb().prepare(`
+      SELECT type, name, sql FROM sqlite_master
+      WHERE name NOT LIKE 'nodes_fts%'
+        AND name NOT IN ('nodes_ai', 'nodes_ad', 'nodes_au')
+      ORDER BY type, name
+    `).all();
+    const expected = nonFtsSchema(control);
+
+    simulateMissingFts5();
+    vi.spyOn(console, 'warn').mockImplementation(() => {});
+    const fallback = initialize();
+
+    expect(fallback.fts5Available).toBe(false);
+    expect(nonFtsSchema(fallback)).toEqual(expected);
+  });
+
+  it.each(['initialization', 'reopening'])('uses real FTS5 after %s', (state) => {
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+    let connection = initialize();
+    expect(connection.fts5Available).toBe(true);
+    new QueryBuilder(connection.getDb()).insertNode(makeNode('loadRecord', 'quasar nebula'));
+
+    if (state === 'reopening') connection = reopen(connection);
+    expect(connection.fts5Available).toBe(true);
+    const queries = new QueryBuilder(connection.getDb());
+    // Only the docstring contains this token: LIKE/fuzzy name search cannot
+    // make this assertion pass if the FTS path is accidentally disabled.
+    expect(queries.searchNodes('nebula').map(result => result.node.name)).toEqual(['loadRecord']);
+    expect(warn).not.toHaveBeenCalled();
+  });
+
+  it('rebuilds real FTS5 after a bulk node load', () => {
+    const connection = initialize();
+    const queries = new QueryBuilder(connection.getDb());
+
+    connection.beginBulkNodeLoad();
+    queries.insertNode(makeNode('loadRecord', 'quasar nebula'));
+    expect(queries.searchNodes('nebula')).toEqual([]);
+    connection.endBulkNodeLoad();
+
+    expect(queries.searchNodes('nebula').map(result => result.node.name)).toEqual(['loadRecord']);
+    queries.insertNode(makeNode('saveRecord', 'pulsar supernova'));
+    expect(queries.searchNodes('supernova').map(result => result.node.name)).toEqual(['saveRecord']);
+  });
+
+  it('repairs an interrupted real FTS5 bulk load on open', () => {
+    let connection = initialize();
+    connection.beginBulkNodeLoad();
+    new QueryBuilder(connection.getDb()).insertNode(makeNode('loadRecord', 'quasar nebula'));
+
+    connection = reopen(connection);
+
+    expect(connection.fts5Available).toBe(true);
+    const queries = new QueryBuilder(connection.getDb());
+    expect(queries.searchNodes('nebula').map(result => result.node.name)).toEqual(['loadRecord']);
+    queries.insertNode(makeNode('saveRecord', 'pulsar supernova'));
+    expect(queries.searchNodes('supernova').map(result => result.node.name)).toEqual(['saveRecord']);
+  });
+});

src/db/index.ts

@@ -83,10 +83,17 @@ export class DatabaseConnection {
    */
   private openedInode: string | null;
 
-  private constructor(db: SqliteDatabase, dbPath: string, backend: SqliteBackend) {
+  /**
+   * Whether FTS5 is available in this Node.js build. When false, search
+   * falls back to LIKE + fuzzy matching (#1532).
+   */
+  readonly fts5Available: boolean;
+
+  private constructor(db: SqliteDatabase, dbPath: string, backend: SqliteBackend, fts5Available: boolean) {
     this.db = db;
     this.dbPath = dbPath;
     this.backend = backend;
+    this.fts5Available = fts5Available;
     this.openedInode = statInode(dbPath);
   }
 
@@ -105,10 +112,41 @@ export class DatabaseConnection {
 
     configureConnection(db);
 
-    // Run schema initialization
+    // Run schema initialization, splitting FTS5 from the rest so
+    // codegraph still works when Node.js was built without FTS5 (#1532).
     const schemaPath = path.join(__dirname, 'schema.sql');
     const schema = fs.readFileSync(schemaPath, 'utf-8');
-    db.exec(schema);
+
+    const FTS5_MARKER = '-- Full-text search index on node names, docstrings, and signatures';
+    const ftsIdx = schema.indexOf(FTS5_MARKER);
+    let fts5Available = true;
+
+    if (ftsIdx >= 0) {
+      const preFts = schema.slice(0, ftsIdx);
+      // FTS ends after the update trigger; required tables and indexes follow
+      // it in schema.sql and must still be created when FTS5 is unavailable.
+      const ftsSection = schema.slice(ftsIdx).match(
+        /^[\s\S]*?CREATE TRIGGER IF NOT EXISTS nodes_au\b[\s\S]*?END;/
+      )?.[0];
+      if (!ftsSection) throw new Error('schema.sql: FTS5 update trigger not found');
+      // Execute everything before FTS5 first
+      db.exec(preFts);
+      // Try FTS5; if it fails, skip it and continue with LIKE-only search
+      try {
+        db.exec(ftsSection);
+      } catch (err: any) {
+        fts5Available = false;
+        const msg = err?.message ?? String(err);
+        console.warn(
+          `[codegraph] FTS5 not available in this Node.js build (${msg}). ` +
+          `Search will fall back to LIKE + fuzzy matching. ` +
+          `For full-text search, use a Node.js build with FTS5 enabled.`
+        );
+      }
+      db.exec(schema.slice(ftsIdx + ftsSection.length));
+    } else {
+      db.exec(schema);
+    }
 
     // Record current schema version so migrations aren't re-applied on open
     const currentVersion = getCurrentVersion(db);
@@ -118,7 +156,7 @@ export class DatabaseConnection {
       ).run(CURRENT_SCHEMA_VERSION, Date.now(), 'Initial schema includes all migrations');
     }
 
-    return new DatabaseConnection(db, dbPath, backend);
+    return new DatabaseConnection(db, dbPath, backend, fts5Available);
   }
 
   /**
@@ -133,8 +171,16 @@ export class DatabaseConnection {
 
     configureConnection(db);
 
+    // Detect FTS5 availability for search fallback (#1532)
+    let fts5Available = true;
+    try {
+      db.exec("SELECT * FROM nodes_fts LIMIT 0");
+    } catch {
+      fts5Available = false;
+    }
+
     // Check and run migrations if needed
-    const conn = new DatabaseConnection(db, dbPath, backend);
+    const conn = new DatabaseConnection(db, dbPath, backend, fts5Available);
     const currentVersion = getCurrentVersion(db);
 
     if (currentVersion < CURRENT_SCHEMA_VERSION) {
@@ -169,6 +215,7 @@ export class DatabaseConnection {
    * row written by anyone during the window is captured by the rebuild.
    */
   beginBulkNodeLoad(): void {
+    if (!this.fts5Available) return;
     for (const t of DatabaseConnection.FTS_TRIGGER_NAMES) {
       this.db.exec(`DROP TRIGGER IF EXISTS ${t}`);
     }
@@ -181,6 +228,7 @@ export class DatabaseConnection {
    * IF NOT EXISTS).
    */
   endBulkNodeLoad(): void {
+    if (!this.fts5Available) return;
     this.db.exec(`INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')`);
     this.recreateFtsTriggers();
   }
@@ -355,6 +403,7 @@ export class DatabaseConnection {
 
   /** Recreate the FTS triggers + rebuild if a bulk-load window never closed. */
   private healBulkNodeLoad(): void {
+    if (!this.fts5Available) return;
     const row = this.db
       .prepare(
         `SELECT count(*) AS c FROM sqlite_master WHERE type = 'trigger' AND name IN ('nodes_ai','nodes_ad','nodes_au')`

src/db/queries.ts

@@ -244,6 +244,9 @@ export class QueryBuilder {
   private projectNameTokens: Set<string> = new Set();
   private isDeprioritizedPath: ((filePath: string) => boolean) | undefined;
 
+  // FTS5 availability flag — detected once at construction time (#1532)
+  private _fts5Available: boolean | undefined;
+
   // Node cache for frequently accessed nodes (LRU-style, max 1000 entries)
   private nodeCache: Map<string, Node> = new Map();
   private readonly maxCacheSize = 1000;
@@ -340,6 +343,13 @@ export class QueryBuilder {
 
   constructor(db: SqliteDatabase) {
     this.db = db;
+    // Detect FTS5 availability once (#1532)
+    try {
+      db.prepare("SELECT * FROM nodes_fts LIMIT 0").get();
+      this._fts5Available = true;
+    } catch {
+      this._fts5Available = false;
+    }
   }
 
   /**
@@ -1300,9 +1310,9 @@ export class QueryBuilder {
     const kinds = mergedKinds;
     const languages = mergedLanguages;
 
-    // First try FTS5 with prefix matching
+    // First try FTS5 with prefix matching (skip if FTS5 not available, #1532)
     let results = text
-      ? this.searchNodesFTS(text, { kinds, languages, limit, offset })
+      ? (this._fts5Available !== false ? this.searchNodesFTS(text, { kinds, languages, limit, offset }) : [])
       // Over-fetch by 5× when running filter-only (no text). The
       // post-scoring path: + name: filters can be very selective, so
       // a smaller multiplier risks returning fewer than `limit`