|
|
@@ -20,7 +20,7 @@ export interface SqliteDatabase {
|
|
|
readonly open: boolean;
|
|
|
}
|
|
|
|
|
|
-export type SqliteBackend = 'native' | 'wasm';
|
|
|
+export type SqliteBackend = 'native' | 'node-sqlite' | 'wasm';
|
|
|
|
|
|
/**
|
|
|
* One-line summary of the recovery steps shown when WASM fallback is
|
|
|
@@ -110,6 +110,72 @@ function resolveParams(params: any[], paramOrder: string[] | null): any {
|
|
|
return params;
|
|
|
}
|
|
|
|
|
|
+/**
|
|
|
+ * Whether an error is SQLite's SQLITE_BUSY / SQLITE_LOCKED ("database is
|
|
|
+ * locked"). Checks better-sqlite3's `code` first, then falls back to message
|
|
|
+ * text for the wasm backend (which throws a plain Error). Exported for tests.
|
|
|
+ */
|
|
|
+export function isDatabaseLockedError(err: unknown): boolean {
|
|
|
+ const code = (err as { code?: unknown } | null)?.code;
|
|
|
+ if (code === 'SQLITE_BUSY' || code === 'SQLITE_LOCKED') return true;
|
|
|
+ const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
|
|
+ return (
|
|
|
+ msg.includes('database is locked') ||
|
|
|
+ msg.includes('database is busy') ||
|
|
|
+ msg.includes('database table is locked') ||
|
|
|
+ msg.includes('sqlite_busy') ||
|
|
|
+ msg.includes('sqlite_locked')
|
|
|
+ );
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Sleep synchronously for `ms` without spinning the CPU. The wasm backend is
|
|
|
+ * single-threaded and synchronous, so an async sleep is useless at the
|
|
|
+ * (synchronous) query call site — we have to actually block this turn while a
|
|
|
+ * writer in another process clears.
|
|
|
+ */
|
|
|
+function sleepSync(ms: number): void {
|
|
|
+ if (ms <= 0) return;
|
|
|
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
|
+}
|
|
|
+
|
|
|
+export interface BusyRetryOptions {
|
|
|
+ /** Total attempts, including the first. */
|
|
|
+ attempts?: number;
|
|
|
+ /** Backoff per retry (ms); the last entry repeats if more retries remain. */
|
|
|
+ backoffMs?: number[];
|
|
|
+ /** Sleep implementation — injectable so tests don't actually wait. */
|
|
|
+ sleep?: (ms: number) => void;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Run a read, retrying on SQLITE_BUSY with bounded backoff.
|
|
|
+ *
|
|
|
+ * Used only by the wasm backend: it can't use WAL (downgraded to DELETE), so a
|
|
|
+ * writer in ANOTHER process (e.g. the git-hook `codegraph sync`) briefly blocks
|
|
|
+ * readers. `busy_timeout` helps but can return immediately when SQLite detects a
|
|
|
+ * would-be deadlock; a short retry rides out the writer. Reads only — never wrap
|
|
|
+ * writes, which run inside transactions guarded by the cross-process FileLock.
|
|
|
+ * The native backend doesn't use this: WAL lets readers proceed during a write.
|
|
|
+ * See issue #238.
|
|
|
+ */
|
|
|
+export function withBusyRetry<T>(fn: () => T, opts: BusyRetryOptions = {}): T {
|
|
|
+ const attempts = opts.attempts ?? 3;
|
|
|
+ const backoff = opts.backoffMs ?? [150, 400];
|
|
|
+ const sleep = opts.sleep ?? sleepSync;
|
|
|
+ let lastErr: unknown;
|
|
|
+ for (let i = 0; i < attempts; i++) {
|
|
|
+ try {
|
|
|
+ return fn();
|
|
|
+ } catch (err) {
|
|
|
+ lastErr = err;
|
|
|
+ if (i === attempts - 1 || !isDatabaseLockedError(err)) throw err;
|
|
|
+ sleep(backoff.length > 0 ? backoff[Math.min(i, backoff.length - 1)]! : 0);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ throw lastErr;
|
|
|
+}
|
|
|
+
|
|
|
/**
|
|
|
* Wraps node-sqlite3-wasm to match the better-sqlite3 interface.
|
|
|
*
|
|
|
@@ -151,12 +217,18 @@ class WasmDatabaseAdapter implements SqliteDatabase {
|
|
|
};
|
|
|
},
|
|
|
get(...params: any[]) {
|
|
|
- const resolved = resolveParams(params, paramOrder);
|
|
|
- return resolved !== undefined ? stmt.get(resolved) : stmt.get();
|
|
|
+ // Reads retry on SQLITE_BUSY — the wasm backend has no WAL, so a writer
|
|
|
+ // in another process can briefly block this read. See issue #238.
|
|
|
+ return withBusyRetry(() => {
|
|
|
+ const resolved = resolveParams(params, paramOrder);
|
|
|
+ return resolved !== undefined ? stmt.get(resolved) : stmt.get();
|
|
|
+ });
|
|
|
},
|
|
|
all(...params: any[]) {
|
|
|
- const resolved = resolveParams(params, paramOrder);
|
|
|
- return resolved !== undefined ? stmt.all(resolved) : stmt.all();
|
|
|
+ return withBusyRetry(() => {
|
|
|
+ const resolved = resolveParams(params, paramOrder);
|
|
|
+ return resolved !== undefined ? stmt.all(resolved) : stmt.all();
|
|
|
+ });
|
|
|
},
|
|
|
};
|
|
|
}
|
|
|
@@ -229,39 +301,175 @@ class WasmDatabaseAdapter implements SqliteDatabase {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * Create a database connection. Tries native better-sqlite3 first,
|
|
|
- * falls back to node-sqlite3-wasm. Returns the active backend
|
|
|
- * alongside the db so each `DatabaseConnection` can report its own
|
|
|
- * backend per-instance — MCP can open multiple project DBs in one
|
|
|
- * process (`tools.ts` getCodeGraph cache), so a process-global would
|
|
|
- * race / overwrite.
|
|
|
+ * Wraps Node's built-in `node:sqlite` (`DatabaseSync`) to match the
|
|
|
+ * better-sqlite3 interface.
|
|
|
+ *
|
|
|
+ * Unlike the wasm adapter this is REAL SQLite compiled into Node, so it supports
|
|
|
+ * WAL, FTS5, mmap, and `@named` params natively — the only shims needed are the
|
|
|
+ * better-sqlite3 conveniences node:sqlite omits: a `.pragma()` helper, a
|
|
|
+ * `.transaction()` helper, and `open` (node:sqlite exposes `isOpen`). It also
|
|
|
+ * needs no statement finalization on close (node-sqlite3-wasm did).
|
|
|
+ *
|
|
|
+ * Available on Node >= 22.5 (the module is simply absent on older Node, so
|
|
|
+ * `createDatabase` falls through to wasm there). The API is still flagged
|
|
|
+ * experimental; `node:sqlite` emits a one-time ExperimentalWarning to stderr on
|
|
|
+ * load, which is harmless for the MCP stdout protocol.
|
|
|
*/
|
|
|
-export function createDatabase(dbPath: string): { db: SqliteDatabase; backend: SqliteBackend } {
|
|
|
- let nativeError: string | undefined;
|
|
|
- let wasmError: string | undefined;
|
|
|
+class NodeSqliteAdapter implements SqliteDatabase {
|
|
|
+ private _db: any;
|
|
|
|
|
|
- // Try native better-sqlite3 first
|
|
|
- try {
|
|
|
+ constructor(dbPath: string) {
|
|
|
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
|
- const Database = require('better-sqlite3');
|
|
|
- const db = new Database(dbPath);
|
|
|
- return { db: db as SqliteDatabase, backend: 'native' };
|
|
|
- } catch (error) {
|
|
|
- nativeError = error instanceof Error ? error.message : String(error);
|
|
|
+ const { DatabaseSync } = require('node:sqlite');
|
|
|
+ this._db = new DatabaseSync(dbPath);
|
|
|
+ }
|
|
|
+
|
|
|
+ get open(): boolean {
|
|
|
+ return this._db.isOpen;
|
|
|
+ }
|
|
|
+
|
|
|
+ prepare(sql: string): SqliteStatement {
|
|
|
+ // node:sqlite matches better-sqlite3's calling convention (variadic
|
|
|
+ // positional args, or a single object for @named params), so params forward
|
|
|
+ // through unchanged — no positional translation like the wasm adapter needs.
|
|
|
+ const stmt = this._db.prepare(sql);
|
|
|
+ return {
|
|
|
+ run(...params: any[]) {
|
|
|
+ const r = stmt.run(...params);
|
|
|
+ return {
|
|
|
+ changes: Number(r?.changes ?? 0),
|
|
|
+ lastInsertRowid: r?.lastInsertRowid ?? 0,
|
|
|
+ };
|
|
|
+ },
|
|
|
+ get(...params: any[]) {
|
|
|
+ return stmt.get(...params);
|
|
|
+ },
|
|
|
+ all(...params: any[]) {
|
|
|
+ return stmt.all(...params);
|
|
|
+ },
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ exec(sql: string): void {
|
|
|
+ this._db.exec(sql);
|
|
|
+ }
|
|
|
+
|
|
|
+ pragma(str: string): any {
|
|
|
+ const trimmed = str.trim();
|
|
|
+ // Write pragma ("key = value"): node:sqlite is real SQLite, so every pragma
|
|
|
+ // (WAL, mmap, synchronous, …) applies as-is — no special-casing like wasm.
|
|
|
+ if (trimmed.includes('=')) {
|
|
|
+ this._db.exec(`PRAGMA ${trimmed}`);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ // Read pragma: return the row object (e.g. { journal_mode: 'wal' }).
|
|
|
+ return this._db.prepare(`PRAGMA ${trimmed}`).get();
|
|
|
+ }
|
|
|
+
|
|
|
+ transaction<T>(fn: (...args: any[]) => T): (...args: any[]) => T {
|
|
|
+ return (...args: any[]) => {
|
|
|
+ this._db.exec('BEGIN');
|
|
|
+ try {
|
|
|
+ const result = fn(...args);
|
|
|
+ this._db.exec('COMMIT');
|
|
|
+ return result;
|
|
|
+ } catch (error) {
|
|
|
+ this._db.exec('ROLLBACK');
|
|
|
+ throw error;
|
|
|
+ }
|
|
|
+ };
|
|
|
}
|
|
|
|
|
|
- // Fall back to WASM
|
|
|
- try {
|
|
|
- const db = new WasmDatabaseAdapter(dbPath);
|
|
|
- console.warn(buildWasmFallbackBanner(nativeError));
|
|
|
- return { db, backend: 'wasm' };
|
|
|
- } catch (error) {
|
|
|
- wasmError = error instanceof Error ? error.message : String(error);
|
|
|
+ close(): void {
|
|
|
+ this._db.close();
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Concise stderr notice shown when better-sqlite3 is unavailable but Node's
|
|
|
+ * built-in node:sqlite is, so we use that instead of the slow wasm fallback.
|
|
|
+ * Unlike wasm, node:sqlite has full WAL + FTS5 and near-native speed, so this is
|
|
|
+ * informational — not a "fix me" warning. Exported for tests.
|
|
|
+ */
|
|
|
+export function buildNodeSqliteNotice(nativeError?: string): string {
|
|
|
+ const lines = [
|
|
|
+ '[CodeGraph] better-sqlite3 unavailable — using the built-in node:sqlite backend.',
|
|
|
+ 'Full WAL + FTS5 support, no native build required. To restore the (fastest)',
|
|
|
+ `native backend: ${WASM_FALLBACK_FIX_RECIPE}`,
|
|
|
+ ];
|
|
|
+ if (nativeError) lines.push(`(better-sqlite3 load error: ${nativeError})`);
|
|
|
+ return lines.join('\n') + '\n';
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Create a database connection, trying backends in order of preference:
|
|
|
+ * 1. better-sqlite3 (native) — fastest, but needs a compiled binding
|
|
|
+ * 2. node:sqlite (Node ≥22.5) — real WAL + FTS5, no native build, no wasm
|
|
|
+ * 3. node-sqlite3-wasm — last resort (no WAL); only ancient Node
|
|
|
+ *
|
|
|
+ * node:sqlite sits ahead of wasm so that when the native binding fails to load
|
|
|
+ * (common on Windows / locked-down CI), users land on a backend WITH WAL instead
|
|
|
+ * of the no-WAL wasm path that causes concurrent-read lock errors (issue #238).
|
|
|
+ *
|
|
|
+ * `CODEGRAPH_SQLITE_BACKEND=native|node-sqlite|wasm` forces a single backend
|
|
|
+ * (used for A/B testing and to opt into node:sqlite); a forced backend that
|
|
|
+ * can't load throws rather than silently falling through.
|
|
|
+ *
|
|
|
+ * Returns the active backend alongside the db so each `DatabaseConnection` can
|
|
|
+ * report its own backend per-instance — MCP can open multiple project DBs in one
|
|
|
+ * process, so a process-global would race / overwrite.
|
|
|
+ */
|
|
|
+export function createDatabase(dbPath: string): { db: SqliteDatabase; backend: SqliteBackend } {
|
|
|
+ const forced = (process.env.CODEGRAPH_SQLITE_BACKEND || '').trim().toLowerCase();
|
|
|
+ const errors: { native?: string; nodeSqlite?: string; wasm?: string } = {};
|
|
|
+ const toMsg = (e: unknown) => (e instanceof Error ? e.message : String(e));
|
|
|
+
|
|
|
+ const tryNative = !forced || forced === 'native';
|
|
|
+ const tryNodeSqlite = !forced || forced === 'node-sqlite' || forced === 'node:sqlite';
|
|
|
+ const tryWasm = !forced || forced === 'wasm';
|
|
|
+
|
|
|
+ // 1. Native better-sqlite3
|
|
|
+ if (tryNative) {
|
|
|
+ try {
|
|
|
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
|
+ const Database = require('better-sqlite3');
|
|
|
+ return { db: new Database(dbPath) as SqliteDatabase, backend: 'native' };
|
|
|
+ } catch (error) {
|
|
|
+ errors.native = toMsg(error);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 2. Node's built-in node:sqlite (real WAL, no native build)
|
|
|
+ if (tryNodeSqlite) {
|
|
|
+ try {
|
|
|
+ const db = new NodeSqliteAdapter(dbPath);
|
|
|
+ // Announce only when this is a genuine fallback (native was tried & failed),
|
|
|
+ // not when the caller explicitly forced node-sqlite.
|
|
|
+ if (!forced && errors.native) {
|
|
|
+ process.stderr.write(buildNodeSqliteNotice(errors.native));
|
|
|
+ }
|
|
|
+ return { db, backend: 'node-sqlite' };
|
|
|
+ } catch (error) {
|
|
|
+ errors.nodeSqlite = toMsg(error);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 3. WASM (no WAL) — last resort
|
|
|
+ if (tryWasm) {
|
|
|
+ try {
|
|
|
+ const db = new WasmDatabaseAdapter(dbPath);
|
|
|
+ console.warn(buildWasmFallbackBanner(errors.native));
|
|
|
+ return { db, backend: 'wasm' };
|
|
|
+ } catch (error) {
|
|
|
+ errors.wasm = toMsg(error);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
throw new Error(
|
|
|
- `Failed to load any SQLite backend.\n` +
|
|
|
- ` Native (better-sqlite3): ${nativeError}\n` +
|
|
|
- ` WASM (node-sqlite3-wasm): ${wasmError}`
|
|
|
+ `Failed to load a SQLite backend.\n` +
|
|
|
+ (errors.native ? ` Native (better-sqlite3): ${errors.native}\n` : '') +
|
|
|
+ (errors.nodeSqlite ? ` node:sqlite: ${errors.nodeSqlite}\n` : '') +
|
|
|
+ (errors.wasm ? ` WASM (node-sqlite3-wasm): ${errors.wasm}\n` : '') +
|
|
|
+ (forced ? ` (CODEGRAPH_SQLITE_BACKEND=${forced} restricted which backends were tried)` : '')
|
|
|
);
|
|
|
}
|