index.ts 65 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717
  1. /**
  2. * CodeGraph
  3. *
  4. * A local-first code intelligence system that builds a semantic
  5. * knowledge graph from any codebase.
  6. */
  7. import * as path from 'path';
  8. import {
  9. Node,
  10. Edge,
  11. FileRecord,
  12. ExtractionResult,
  13. Subgraph,
  14. TraversalOptions,
  15. SearchOptions,
  16. SearchResult,
  17. SegmentMatch,
  18. Context,
  19. GraphStats,
  20. TaskInput,
  21. TaskContext,
  22. BuildContextOptions,
  23. FindRelevantContextOptions,
  24. } from './types';
  25. import { DatabaseConnection, getDatabasePath, removeDatabaseFiles } from './db';
  26. import { WalCheckpointValve } from './db/wal-valve';
  27. import { QueryBuilder } from './db/queries';
  28. import {
  29. isInitialized,
  30. createDirectory,
  31. removeDirectory,
  32. validateDirectory,
  33. } from './directory';
  34. import {
  35. ExtractionOrchestrator,
  36. IndexProgress,
  37. IndexResult,
  38. SyncResult,
  39. extractFromSource,
  40. initGrammars,
  41. } from './extraction';
  42. import {
  43. ReferenceResolver,
  44. createResolver,
  45. ResolutionResult,
  46. } from './resolution';
  47. import { GraphTraverser, GraphQueryManager } from './graph';
  48. import { ContextBuilder, createContextBuilder } from './context';
  49. import { Mutex, FileLock } from './utils';
  50. import { FileWatcher, WatchOptions, PendingFile, LockUnavailableError } from './sync';
  51. import { EXTRACTION_VERSION } from './extraction/extraction-version';
  52. import { getCodeGraphDir } from './directory';
  53. import { deriveProjectNameTokens } from './search/query-utils';
  54. import { CodeGraphPackageVersion } from './mcp/version';
  55. import { segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments';
  56. import { createYielder } from './resolution/cooperative-yield';
  57. import { minRefsForPool } from './resolution/resolver-pool';
  58. // Re-export types for consumers
  59. export * from './types';
  60. // Storage building blocks for embedded/SDK consumers that drive the graph
  61. // directly (open a DB, run prepared queries) rather than through the CodeGraph
  62. // facade. Exposed from the package entry so they no longer require deep imports
  63. // into dist/ (issue #354).
  64. export { getDatabasePath, DatabaseConnection } from './db';
  65. export { QueryBuilder } from './db/queries';
  66. export {
  67. getCodeGraphDir,
  68. isInitialized,
  69. findNearestCodeGraphRoot,
  70. CODEGRAPH_DIR,
  71. } from './directory';
  72. export { IndexProgress, IndexResult, SyncResult } from './extraction';
  73. export { detectLanguage, isLanguageSupported, isGrammarLoaded, getSupportedLanguages, initGrammars, loadGrammarsForLanguages, loadAllGrammars } from './extraction';
  74. export { ResolutionResult } from './resolution';
  75. export {
  76. CodeGraphError,
  77. FileError,
  78. ParseError,
  79. DatabaseError,
  80. SearchError,
  81. VectorError,
  82. ConfigError,
  83. Logger,
  84. setLogger,
  85. getLogger,
  86. silentLogger,
  87. defaultLogger,
  88. } from './errors';
  89. export { Mutex, FileLock, processInBatches, debounce, throttle, MemoryMonitor } from './utils';
  90. export { FileWatcher, WatchOptions, PendingFile, LockUnavailableError } from './sync';
  91. export { MCPServer } from './mcp';
  92. /**
  93. * Options for initializing a new CodeGraph project
  94. */
  95. export interface InitOptions {
  96. /** Whether to run initial indexing after init */
  97. index?: boolean;
  98. /** Progress callback for indexing */
  99. onProgress?: (progress: IndexProgress) => void;
  100. }
  101. /**
  102. * Options for opening an existing CodeGraph project
  103. */
  104. export interface OpenOptions {
  105. /** Whether to run sync if files have changed */
  106. sync?: boolean;
  107. /** Whether to run in read-only mode */
  108. readOnly?: boolean;
  109. }
  110. /**
  111. * Options for indexing
  112. */
  113. export interface IndexOptions {
  114. /** Progress callback */
  115. onProgress?: (progress: IndexProgress) => void;
  116. /** Abort signal for cancellation */
  117. signal?: AbortSignal;
  118. /** Enable verbose logging (worker lifecycle, memory, timeouts) */
  119. verbose?: boolean;
  120. }
  121. /**
  122. * Main CodeGraph class
  123. *
  124. * Provides the primary interface for interacting with the code knowledge graph.
  125. */
  126. export class CodeGraph {
  127. private db: DatabaseConnection;
  128. private queries: QueryBuilder;
  129. private projectRoot: string;
  130. // Assigned via wireLayers() from the constructor (and again on reopen) — the
  131. // `!` tells TS these are definitely set even though the assignment is one
  132. // method call away from the constructor body.
  133. private orchestrator!: ExtractionOrchestrator;
  134. private resolver!: ReferenceResolver;
  135. private graphManager!: GraphQueryManager;
  136. private traverser!: GraphTraverser;
  137. private contextBuilder!: ContextBuilder;
  138. // Mutex for preventing concurrent indexing operations (in-process)
  139. private indexMutex = new Mutex();
  140. // File lock for preventing concurrent writes across processes (CLI, MCP, git hooks)
  141. private fileLock: FileLock;
  142. // File watcher for auto-sync on file changes
  143. private watcher: FileWatcher | null = null;
  144. private constructor(
  145. db: DatabaseConnection,
  146. queries: QueryBuilder,
  147. projectRoot: string
  148. ) {
  149. this.db = db;
  150. this.queries = queries;
  151. this.projectRoot = projectRoot;
  152. this.fileLock = new FileLock(
  153. path.join(getCodeGraphDir(projectRoot), 'codegraph.lock')
  154. );
  155. this.wireLayers();
  156. }
  157. /**
  158. * (Re)build the query/extraction/graph layers over the current `this.queries`
  159. * (which wraps `this.db`). Factored out of the constructor so `reopenIfReplaced`
  160. * can rebuild them against a fresh connection without duplicating the wiring.
  161. * The path-based `fileLock` is independent of the DB handle, so it stays put.
  162. */
  163. private wireLayers(): void {
  164. // Down-weight the project name as a query term in search ranking — it names
  165. // the whole repo, not a symbol, so it has no discriminative value (#720).
  166. try {
  167. this.queries.setProjectNameTokens(deriveProjectNameTokens(this.projectRoot));
  168. } catch {
  169. // Best-effort: ranking still works without it.
  170. }
  171. this.orchestrator = new ExtractionOrchestrator(this.projectRoot, this.queries);
  172. this.resolver = createResolver(this.projectRoot, this.queries);
  173. this.graphManager = new GraphQueryManager(this.queries);
  174. this.traverser = new GraphTraverser(this.queries);
  175. this.contextBuilder = createContextBuilder(
  176. this.projectRoot,
  177. this.queries,
  178. this.traverser
  179. );
  180. }
  181. /**
  182. * Heal a stale database handle in place. If `.codegraph/` was removed and
  183. * recreated at the SAME path while this instance held the DB open — a git
  184. * worktree removed and re-added, or `rm -rf .codegraph` + `codegraph init` —
  185. * our open fd points at the now-unlinked inode and can never see the new
  186. * index, so every query returns the pre-removal snapshot until the process
  187. * restarts (#925). When that's detected, open the live file at the same path,
  188. * rebuild the query layers, and swap them IN PLACE, so every holder of this
  189. * instance (the MCP daemon's default project, cached projectPath connections)
  190. * heals without a restart. Returns true iff it reopened.
  191. *
  192. * POSIX-only in practice: `isReplacedOnDisk` never fires on Windows (an open
  193. * file can't be unlinked there, and st_ino is unreliable).
  194. */
  195. reopenIfReplaced(): boolean {
  196. if (!this.db.isReplacedOnDisk()) return false;
  197. const dbPath = this.db.getPath();
  198. // Open the live file FIRST — if that throws (e.g. mid-recreate), the old
  199. // handle stays in place and the caller retries on the next query, rather
  200. // than leaving this instance with no connection at all.
  201. const fresh = DatabaseConnection.open(dbPath);
  202. const stale = this.db;
  203. this.db = fresh;
  204. this.queries = new QueryBuilder(fresh.getDb());
  205. this.wireLayers();
  206. // Releasing the dead handle also frees the leaked db/-wal/-shm fds that were
  207. // pinning the unlinked inode (#925).
  208. try { stale.close(); } catch { /* the old inode is gone; closing just frees fds */ }
  209. return true;
  210. }
  211. // ===========================================================================
  212. // Lifecycle Methods
  213. // ===========================================================================
  214. /**
  215. * Initialize a new CodeGraph project
  216. *
  217. * Creates the .CodeGraph directory, database, and configuration.
  218. *
  219. * @param projectRoot - Path to the project root directory
  220. * @param options - Initialization options
  221. * @returns A new CodeGraph instance
  222. */
  223. static async init(projectRoot: string, options: InitOptions = {}): Promise<CodeGraph> {
  224. await initGrammars();
  225. const resolvedRoot = path.resolve(projectRoot);
  226. // Check if already initialized
  227. if (isInitialized(resolvedRoot)) {
  228. throw new Error(`CodeGraph already initialized in ${resolvedRoot}`);
  229. }
  230. // Create directory structure
  231. createDirectory(resolvedRoot);
  232. // Initialize database
  233. const dbPath = getDatabasePath(resolvedRoot);
  234. const db = DatabaseConnection.initialize(dbPath);
  235. const queries = new QueryBuilder(db.getDb());
  236. const instance = new CodeGraph(db, queries, resolvedRoot);
  237. // Run initial indexing if requested
  238. if (options.index) {
  239. await instance.indexAll({ onProgress: options.onProgress });
  240. }
  241. return instance;
  242. }
  243. /**
  244. * Initialize synchronously (without indexing)
  245. */
  246. static initSync(projectRoot: string): CodeGraph {
  247. const resolvedRoot = path.resolve(projectRoot);
  248. // Check if already initialized
  249. if (isInitialized(resolvedRoot)) {
  250. throw new Error(`CodeGraph already initialized in ${resolvedRoot}`);
  251. }
  252. // Create directory structure
  253. createDirectory(resolvedRoot);
  254. // Initialize database
  255. const dbPath = getDatabasePath(resolvedRoot);
  256. const db = DatabaseConnection.initialize(dbPath);
  257. const queries = new QueryBuilder(db.getDb());
  258. return new CodeGraph(db, queries, resolvedRoot);
  259. }
  260. /**
  261. * Open an existing CodeGraph project
  262. *
  263. * @param projectRoot - Path to the project root directory
  264. * @param options - Open options
  265. * @returns A CodeGraph instance
  266. */
  267. static async open(projectRoot: string, options: OpenOptions = {}): Promise<CodeGraph> {
  268. await initGrammars();
  269. const resolvedRoot = path.resolve(projectRoot);
  270. // Check if initialized
  271. if (!isInitialized(resolvedRoot)) {
  272. throw new Error(`CodeGraph not initialized in ${resolvedRoot}. Run init() first.`);
  273. }
  274. // Validate directory structure
  275. const validation = validateDirectory(resolvedRoot);
  276. if (!validation.valid) {
  277. throw new Error(`Invalid CodeGraph directory: ${validation.errors.join(', ')}`);
  278. }
  279. // Open database
  280. const dbPath = getDatabasePath(resolvedRoot);
  281. const db = DatabaseConnection.open(dbPath);
  282. const queries = new QueryBuilder(db.getDb());
  283. const instance = new CodeGraph(db, queries, resolvedRoot);
  284. // Sync if requested
  285. if (options.sync) {
  286. await instance.sync();
  287. }
  288. return instance;
  289. }
  290. /**
  291. * Rebuild the project's database from scratch and return a fresh, empty
  292. * instance — the "same result as a fresh init" semantics that `codegraph
  293. * index` documents.
  294. *
  295. * Unlike `open()` followed by `clear()`, this DISCARDS the existing
  296. * `.codegraph/codegraph.db` (and its `-wal`/`-shm` sidecars) before
  297. * re-initializing, instead of opening the old database and DELETE-ing every
  298. * row. On a large or pre-fix poisoned index — e.g. an old graph that scanned
  299. * an ignored gitlink corpus (#1065) into ~1.6M nodes with a multi-GB WAL —
  300. * the per-row `nodes_fts` delete-trigger churn blocks the main thread long
  301. * enough to trip the #850 liveness watchdog before indexing even starts, so a
  302. * full re-index could never recover the bad state (#1067). Discarding the
  303. * files is O(1) regardless of size, reclaims the disk, and sidesteps opening
  304. * (and running migrations against) the poisoned database entirely.
  305. */
  306. static async recreate(projectRoot: string): Promise<CodeGraph> {
  307. await initGrammars();
  308. const resolvedRoot = path.resolve(projectRoot);
  309. // Check if initialized — recreate REBUILDS an existing project; it is not a
  310. // first-time `init`.
  311. if (!isInitialized(resolvedRoot)) {
  312. throw new Error(`CodeGraph not initialized in ${resolvedRoot}. Run init() first.`);
  313. }
  314. const dbPath = getDatabasePath(resolvedRoot);
  315. try {
  316. removeDatabaseFiles(dbPath);
  317. } catch (err) {
  318. // POSIX unlinks an open file fine; this fires mainly on Windows when a
  319. // live daemon/MCP server still holds the database. Turn the raw EBUSY into
  320. // an actionable instruction instead of a generic failure.
  321. const reason = err instanceof Error ? err.message : String(err);
  322. throw new Error(
  323. `Could not rebuild the index — the database file is in use (${reason}). ` +
  324. `Stop any running CodeGraph MCP server/daemon for this project and retry, ` +
  325. `or remove the ${getCodeGraphDir(resolvedRoot)} directory and run "codegraph init".`
  326. );
  327. }
  328. // Re-create an empty, freshly-schema'd database at the same path.
  329. const db = DatabaseConnection.initialize(dbPath);
  330. const queries = new QueryBuilder(db.getDb());
  331. return new CodeGraph(db, queries, resolvedRoot);
  332. }
  333. /**
  334. * Open synchronously (without sync)
  335. */
  336. static openSync(projectRoot: string): CodeGraph {
  337. const resolvedRoot = path.resolve(projectRoot);
  338. // Check if initialized
  339. if (!isInitialized(resolvedRoot)) {
  340. throw new Error(`CodeGraph not initialized in ${resolvedRoot}. Run init() first.`);
  341. }
  342. // Validate directory structure
  343. const validation = validateDirectory(resolvedRoot);
  344. if (!validation.valid) {
  345. throw new Error(`Invalid CodeGraph directory: ${validation.errors.join(', ')}`);
  346. }
  347. // Open database
  348. const dbPath = getDatabasePath(resolvedRoot);
  349. const db = DatabaseConnection.open(dbPath);
  350. const queries = new QueryBuilder(db.getDb());
  351. return new CodeGraph(db, queries, resolvedRoot);
  352. }
  353. /**
  354. * Check if a directory has been initialized as a CodeGraph project
  355. */
  356. static isInitialized(projectRoot: string): boolean {
  357. return isInitialized(path.resolve(projectRoot));
  358. }
  359. /**
  360. * Close the CodeGraph instance and release resources
  361. */
  362. close(): void {
  363. this.unwatch();
  364. // Release file lock if held
  365. this.fileLock.release();
  366. this.db.close();
  367. }
  368. /**
  369. * Get the project root directory
  370. */
  371. getProjectRoot(): string {
  372. return this.projectRoot;
  373. }
  374. // ===========================================================================
  375. // Indexing
  376. // ===========================================================================
  377. /**
  378. * Index all files in the project
  379. *
  380. * Uses a mutex to prevent concurrent indexing operations.
  381. */
  382. async indexAll(options: IndexOptions = {}): Promise<IndexResult> {
  383. return this.indexMutex.withLock(async () => {
  384. try {
  385. this.fileLock.acquire();
  386. } catch {
  387. return { success: false, filesIndexed: 0, filesSkipped: 0, filesErrored: 0, nodesCreated: 0, edgesCreated: 0, errors: [{ message: 'Could not acquire file lock - another process may be indexing', severity: 'error' as const }], durationMs: 0 };
  388. }
  389. // Defer WAL auto-checkpointing for the whole bulk run (#1231): the
  390. // default 1000-page interval re-writes hot pages into the main DB file
  391. // over and over — ~95% of all disk I/O during a bulk index, and a
  392. // 19+min → 45s difference on HDD-class storage. The valve bounds WAL
  393. // growth by backfilling PASSIVEly on a worker thread (never blocking
  394. // the writer or the #850 watchdog heartbeat); runMaintenance below does
  395. // the final fold-up before the interval is restored in the finally.
  396. // Kill switch: CODEGRAPH_NO_WAL_DEFER=1. Non-WAL journal modes (some
  397. // network filesystems) have no WAL to defer — skip.
  398. // Fast-init: on a COMPLETELY fresh DB, trade crash-durability for speed
  399. // during the bulk build (journal in memory, no fsync). Safe because the
  400. // DB is disposable until the index completes — index_state stays
  401. // 'indexing' and a crashed init is re-run from scratch; existing DBs
  402. // (re-index/sync) never take this path. Kill switch:
  403. // CODEGRAPH_NO_FAST_INIT=1 (same pattern as CODEGRAPH_NO_WAL_DEFER).
  404. const freshDb = this.queries.getNodeAndEdgeCount().nodes === 0;
  405. const fastInit = process.env.CODEGRAPH_NO_FAST_INIT !== '1' && freshDb;
  406. if (fastInit) {
  407. try {
  408. this.db.getDb().pragma('journal_mode = MEMORY');
  409. this.db.getDb().pragma('synchronous = OFF');
  410. } catch { /* keep WAL */ }
  411. }
  412. const deferWal = !fastInit && process.env.CODEGRAPH_NO_WAL_DEFER !== '1' && this.db.getJournalMode() === 'wal';
  413. let walValve: WalCheckpointValve | null = null;
  414. let priorAutocheckpoint = 1000;
  415. if (deferWal) {
  416. priorAutocheckpoint = this.db.getWalAutocheckpoint();
  417. this.db.setWalAutocheckpoint(0);
  418. walValve = new WalCheckpointValve(
  419. this.db,
  420. undefined,
  421. undefined,
  422. options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
  423. );
  424. walValve.start();
  425. }
  426. try {
  427. const before = this.queries.getNodeAndEdgeCount();
  428. // Mark the index as in-flight BEFORE any writes: a run killed
  429. // mid-index (OOM, SIGKILL, the #850 liveness watchdog) leaves this
  430. // marker behind, so `codegraph status` can tell a truncated index
  431. // from a completed one instead of silently serving partial results.
  432. try { this.queries.setMetadata('index_state', 'indexing'); } catch { /* metadata is advisory */ }
  433. // Segment vocabulary starts empty and is repopulated by the node write
  434. // path as every file (re-)indexes below — so a full index is also the
  435. // orphan-cleanup pass for names deleted since the last one.
  436. try { this.queries.clearNameSegmentVocab(); } catch { /* vocab is advisory — never fail an index over it */ }
  437. // Bulk FTS mode for the mass-insert phase: drop the per-row FTS sync
  438. // triggers, rebuild nodes_fts once from the nodes table afterwards.
  439. // Crash inside the window is healed on the next DatabaseConnection.open.
  440. this.db.beginBulkNodeLoad();
  441. let result: IndexResult;
  442. try {
  443. result = await this.orchestrator.indexAll(
  444. options.onProgress,
  445. options.signal,
  446. options.verbose,
  447. walValve ? () => walValve!.backpressure() : undefined,
  448. // Store-writer offload is fresh-DB-only: with any pre-existing
  449. // data the store path must read (existing-file checks, cross-file
  450. // edge snapshots) and delete, which belongs on one thread.
  451. freshDb ? { dbPath: this.db.getPath(), fastInit } : null
  452. );
  453. } finally {
  454. this.db.endBulkNodeLoad();
  455. }
  456. // Fold the parse phase's WAL BEFORE the first post-parse reads
  457. // (resolver re-init and resolution both read on the main thread):
  458. // paging a bulk-write-sized WAL there is what blew the #850
  459. // watchdog's 60s window in the #1231 repro. Off-thread + awaited,
  460. // so the event loop keeps turning.
  461. if (walValve) await walValve.foldNow();
  462. // Re-detect frameworks now that the index is populated. The resolver
  463. // is constructed with createResolver() before any files exist, so
  464. // framework resolvers whose detect() consults the indexed file list
  465. // (e.g. UIKit/SwiftUI scanning for imports, swift-objc-bridge looking
  466. // for both Swift and ObjC files) all return false on that initial pass
  467. // and silently drop themselves. Re-initializing here gives them a
  468. // chance to see the actual project before resolution runs.
  469. if (result.success && result.filesIndexed > 0) {
  470. this.resolver.initialize();
  471. // Cross-file finalization (e.g. NestJS RouterModule prefixes). Runs
  472. // before resolution so updated names show up in subsequent reads.
  473. this.resolver.runPostExtract();
  474. }
  475. // Resolve references to create call/import/extends edges
  476. if (result.success && result.filesIndexed > 0) {
  477. // Get count without loading all refs into memory
  478. const unresolvedCount = this.queries.getUnresolvedReferencesCount();
  479. // Fast-init leaves the DB in memory-journal (rollback) mode, where
  480. // the parallel resolver pool's read connections would contend with
  481. // the main writer's exclusive commits. When the pool will actually
  482. // run (enough pending refs), restore WAL BEFORE resolution so
  483. // readers never block the writer; otherwise stay in the fast mode
  484. // until the finally — sequential resolution has no readers.
  485. if (fastInit && unresolvedCount >= minRefsForPool()) {
  486. try {
  487. this.db.getDb().pragma('synchronous = NORMAL');
  488. this.db.getDb().pragma('journal_mode = WAL');
  489. } catch { /* keep current mode; resolution still works sequentially */ }
  490. }
  491. options.onProgress?.({
  492. phase: 'resolving',
  493. current: 0,
  494. total: unresolvedCount,
  495. });
  496. await this.resolveReferencesBatched(
  497. (current, total) => {
  498. options.onProgress?.({
  499. phase: 'resolving',
  500. current,
  501. total,
  502. });
  503. },
  504. (done, totalPasses) => {
  505. options.onProgress?.({
  506. phase: 'linking',
  507. current: done,
  508. total: totalPasses,
  509. });
  510. }
  511. );
  512. // Second pass: chained calls whose method lives on a supertype the
  513. // receiver conforms to (protocol-extension / inherited / default-
  514. // interface). Needs the implements/extends edges the main pass just
  515. // built, so it runs after resolution (#750).
  516. const tChained = Date.now();
  517. await this.resolver.resolveChainedCallsViaConformance();
  518. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[synth-timing] chainedConformance: ${Date.now() - tChained}ms`);
  519. // Same lifecycle for `this.<member>` callback registrations whose
  520. // member is inherited from a supertype (#808).
  521. const tDeferred = Date.now();
  522. await this.resolver.resolveDeferredThisMemberRefs();
  523. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[synth-timing] deferredThisMember: ${Date.now() - tDeferred}ms`);
  524. }
  525. // Refresh planner stats + checkpoint the WAL after bulk writes.
  526. // Off-thread (worker connection): on a multi-GB index this is minutes
  527. // of IO, and inline it starved the #850 watchdog AFTER a fully
  528. // successful index. Never load-bearing for correctness.
  529. if (result.success && result.filesIndexed > 0) {
  530. const tMaint = Date.now();
  531. // Quiesce the valve first so its in-flight checkpoint and the
  532. // maintenance checkpoint don't contend for the checkpointer lock
  533. // (the loser would silently no-op and leave the WAL unfolded).
  534. if (walValve) { walValve.stop(); await walValve.drain(); }
  535. await this.db.runMaintenance();
  536. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] maintenance: ${Date.now() - tMaint}ms`);
  537. }
  538. // The orchestrator only sees extraction-phase counts; resolution and
  539. // synthesizer edges (often >50% of the graph on JVM repos) come later.
  540. // Recompute against the DB so the CLI summary reports the true totals.
  541. if (result.success && result.filesIndexed > 0) {
  542. const tCount = Date.now();
  543. const after = this.queries.getNodeAndEdgeCount();
  544. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] count-recompute: ${Date.now() - tCount}ms`);
  545. result.nodesCreated = after.nodes - before.nodes;
  546. result.edgesCreated = after.edges - before.edges;
  547. }
  548. // Stamp the index with the engine that built it, so `codegraph status`
  549. // and `codegraph upgrade` can recommend a re-index when the running
  550. // engine produces richer extraction than the one on disk. Only on a
  551. // real full index — a sync touches a subset, so it must NOT advance the
  552. // extraction stamp (the bulk would still be stale). See extraction-version.ts.
  553. if (result.success && result.filesIndexed > 0) {
  554. try {
  555. this.queries.setMetadata('indexed_with_version', CodeGraphPackageVersion);
  556. this.queries.setMetadata('indexed_with_extraction_version', String(EXTRACTION_VERSION));
  557. } catch { /* metadata is advisory — never fail an index over it */ }
  558. }
  559. // Reconcile the scan's ground truth against what the pipeline
  560. // accounted for. A shortfall means files were silently dropped
  561. // (observed in the wild: a run under heavy load came up 37 files
  562. // short with no error) — record it and tell the user, don't let the
  563. // index pass as complete.
  564. try {
  565. if (!result.success) {
  566. this.queries.setMetadata('index_state', 'failed');
  567. } else {
  568. const accounted = result.filesIndexed + result.filesSkipped + result.filesErrored;
  569. const discovered = result.filesDiscovered;
  570. const shortfall = discovered !== undefined ? discovered - accounted : 0;
  571. if (discovered !== undefined && shortfall > 0) {
  572. this.queries.setMetadata('index_state', 'partial');
  573. this.queries.setMetadata('index_files_discovered', String(discovered));
  574. this.queries.setMetadata('index_files_accounted', String(accounted));
  575. result.errors.push({
  576. message: `Index is missing ${shortfall} of ${discovered} discovered files (indexed ${result.filesIndexed}, skipped ${result.filesSkipped}, errored ${result.filesErrored}). The index is PARTIAL — re-run \`codegraph index\`.`,
  577. severity: 'warning',
  578. code: 'index_partial',
  579. });
  580. } else {
  581. this.queries.setMetadata('index_state', 'complete');
  582. if (discovered !== undefined) {
  583. this.queries.setMetadata('index_files_discovered', String(discovered));
  584. this.queries.setMetadata('index_files_accounted', String(accounted));
  585. }
  586. }
  587. }
  588. } catch { /* metadata is advisory — never fail an index over it */ }
  589. return result;
  590. } finally {
  591. // Restore the auto-checkpoint interval AFTER the fold-up above so the
  592. // next ordinary write doesn't inherit a giant inline checkpoint. On
  593. // the error path the WAL may still be large; correctness is unchanged
  594. // (SQLite replays the WAL on the next open) and the follow-up write
  595. // that folds it is the known cost of a failed run.
  596. if (walValve) { walValve.stop(); await walValve.drain(); }
  597. if (deferWal) {
  598. try { this.db.setWalAutocheckpoint(priorAutocheckpoint); } catch { /* connection may be closing */ }
  599. }
  600. if (fastInit) {
  601. // Back to the durable defaults; journal_mode=WAL folds the MEMORY
  602. // journal state into a normal WAL-mode database file.
  603. try {
  604. this.db.getDb().pragma('synchronous = NORMAL');
  605. this.db.getDb().pragma('journal_mode = WAL');
  606. } catch { /* connection may be closing */ }
  607. }
  608. this.fileLock.release();
  609. }
  610. });
  611. }
  612. /**
  613. * Index specific files
  614. *
  615. * Uses a mutex to prevent concurrent indexing operations.
  616. */
  617. async indexFiles(filePaths: string[]): Promise<IndexResult> {
  618. return this.indexMutex.withLock(async () => {
  619. try {
  620. this.fileLock.acquire();
  621. } catch {
  622. return { success: false, filesIndexed: 0, filesSkipped: 0, filesErrored: 0, nodesCreated: 0, edgesCreated: 0, errors: [{ message: 'Could not acquire file lock - another process may be indexing', severity: 'error' as const }], durationMs: 0 };
  623. }
  624. try {
  625. return this.orchestrator.indexFiles(filePaths);
  626. } finally {
  627. this.fileLock.release();
  628. }
  629. });
  630. }
  631. /**
  632. * Sync with current file state (incremental update)
  633. *
  634. * Uses a mutex to prevent concurrent indexing operations.
  635. */
  636. async sync(options: IndexOptions = {}): Promise<SyncResult> {
  637. return this.indexMutex.withLock(async () => {
  638. try {
  639. this.fileLock.acquire();
  640. } catch {
  641. return { filesChecked: 0, filesAdded: 0, filesModified: 0, filesRemoved: 0, nodesUpdated: 0, durationMs: 0 };
  642. }
  643. try {
  644. // Captured BEFORE the sync runs: the sync's own incremental writes
  645. // populate vocab rows for the files it touches, so an end-of-sync
  646. // emptiness check would see "non-empty" and skip the backfill forever,
  647. // leaving every unchanged file's names unsegmented.
  648. const vocabWasEmpty = (() => {
  649. try { return this.queries.isNameSegmentVocabEmpty(); } catch { return false; }
  650. })();
  651. const result = await this.orchestrator.sync(options.onProgress);
  652. // Cross-file finalization (e.g. NestJS RouterModule prefixes). Run on
  653. // every sync that touched files so edits to `app.module.ts` propagate
  654. // to controllers in unchanged files. The pass is idempotent and cheap
  655. // (regex over *.module.ts only).
  656. if (result.filesAdded > 0 || result.filesModified > 0) {
  657. this.resolver.runPostExtract();
  658. } else if (result.filesRemoved > 0) {
  659. // A pure-removal sync still resolves refs below — the deletion path
  660. // resurrects the removed file's incoming edges as pending refs
  661. // (#1240 removal case) and the orphan sweep consumes them. In a
  662. // long-lived process (daemon) the resolver's name caches were
  663. // warmed against the pre-removal graph; drop them so resolution
  664. // sees the post-removal state. (runPostExtract above clears caches
  665. // itself, so the changed-files branch is already covered.)
  666. this.resolver.clearCaches();
  667. }
  668. // Resolve references if files were updated
  669. const filesChanged = result.filesAdded > 0 || result.filesModified > 0;
  670. if (filesChanged) {
  671. if (result.changedFilePaths) {
  672. // Scope resolution to changed files (git fast path — bounded set)
  673. const tRefLoad = Date.now();
  674. const unresolvedRefs = this.queries.getUnresolvedReferencesByFiles(result.changedFilePaths);
  675. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-ref-load: ${Date.now() - tRefLoad}ms (${unresolvedRefs.length} refs)`);
  676. options.onProgress?.({
  677. phase: 'resolving',
  678. current: 0,
  679. total: unresolvedRefs.length,
  680. });
  681. this.resolver.resolveAndPersist(unresolvedRefs, (current, total) => {
  682. options.onProgress?.({
  683. phase: 'resolving',
  684. current,
  685. total,
  686. });
  687. });
  688. // Retry previously-failed refs the changed files may now satisfy
  689. // (#1240). Scoped resolution above only re-resolves refs FROM the
  690. // changed files — but when a changed file gains an export/symbol,
  691. // refs in UNCHANGED files that failed against the old graph can
  692. // now resolve, and nothing else ever revisits them (their rows
  693. // were parked as status='failed' by an earlier completed pass).
  694. // Look them up by the symbol names the changed files now carry
  695. // and re-resolve just that set. On a sync where no failed ref
  696. // matches, this is one indexed lookup.
  697. const tRetry = Date.now();
  698. const retryable = this.queries.getRetryableFailedReferences(
  699. this.queries.getNodeNamesByFiles(result.changedFilePaths)
  700. );
  701. if (retryable.length > 0) {
  702. options.onProgress?.({
  703. phase: 'resolving',
  704. current: 0,
  705. total: retryable.length,
  706. });
  707. await this.resolver.resolveAndPersistListYielding(retryable);
  708. options.onProgress?.({
  709. phase: 'resolving',
  710. current: retryable.length,
  711. total: retryable.length,
  712. });
  713. }
  714. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-failed-ref-retry: ${Date.now() - tRetry}ms (${retryable.length} refs)`);
  715. } else {
  716. // No git info — use batched resolution to avoid OOM
  717. const unresolvedCount = this.queries.getUnresolvedReferencesCount();
  718. options.onProgress?.({
  719. phase: 'resolving',
  720. current: 0,
  721. total: unresolvedCount,
  722. });
  723. await this.resolveReferencesBatched(
  724. (current, total) => {
  725. options.onProgress?.({
  726. phase: 'resolving',
  727. current,
  728. total,
  729. });
  730. },
  731. (done, totalPasses) => {
  732. options.onProgress?.({
  733. phase: 'linking',
  734. current: done,
  735. total: totalPasses,
  736. });
  737. }
  738. );
  739. }
  740. }
  741. // Orphan sweep (#1187). A resolution pass that dies mid-run — the #850
  742. // daemon liveness watchdog's SIGKILL (#1122), Ctrl-C, a crash — leaves
  743. // the refs it never reached in unresolved_refs, and the git-scoped fast
  744. // path above never revisits them (it reads only the changed files'
  745. // rows). Those files' call edges were then missing PERMANENTLY, with
  746. // nothing to see except a too-small blast radius, until a full
  747. // re-index. A completed pass takes every row it processed out of the
  748. // PENDING set (resolved rows are deleted, unresolvable ones parked as
  749. // status='failed' for the #1240 retry above), so any pending row now
  750. // is such an orphan — or a row from an older engine's scoped pass.
  751. // Grind them down with the batched resolver; this also makes a bare
  752. // `codegraph sync` the recovery command for a wedged index. On a
  753. // healthy index this is one COUNT query.
  754. const orphanCount = this.queries.getUnresolvedReferencesCount();
  755. if (orphanCount > 0) {
  756. options.onProgress?.({
  757. phase: 'resolving',
  758. current: 0,
  759. total: orphanCount,
  760. });
  761. await this.resolveReferencesBatched(
  762. (current, total) => {
  763. options.onProgress?.({
  764. phase: 'resolving',
  765. current,
  766. total,
  767. });
  768. },
  769. (done, totalPasses) => {
  770. options.onProgress?.({
  771. phase: 'linking',
  772. current: done,
  773. total: totalPasses,
  774. });
  775. }
  776. );
  777. }
  778. if (filesChanged || orphanCount > 0) {
  779. // Second pass: chained calls whose method lives on a supertype the
  780. // receiver conforms to (protocol-extension / inherited). Needs the
  781. // implements/extends edges built above (#750).
  782. await this.resolver.resolveChainedCallsViaConformance();
  783. // Same lifecycle for `this.<member>` callback registrations whose
  784. // member is inherited from a supertype (#808).
  785. await this.resolver.resolveDeferredThisMemberRefs();
  786. }
  787. // Refresh planner stats + checkpoint the WAL after bulk writes.
  788. // Off-thread — see indexAll's call site.
  789. if (filesChanged || result.filesRemoved > 0 || orphanCount > 0) {
  790. await this.db.runMaintenance();
  791. }
  792. // Heal the segment vocabulary on indexes built before the table
  793. // existed (upgrade path): incremental writes above only cover changed
  794. // files, so a vocab that was empty when this sync STARTED means the
  795. // bulk was never segmented — backfill it (INSERT OR IGNORE, so the
  796. // rows the sync just wrote are fine). Batched + yielding — sync can
  797. // run on the daemon's liveness-watchdog thread (#850/#1091).
  798. try {
  799. if (vocabWasEmpty && this.queries.getNodeAndEdgeCount().nodes > 0) {
  800. await this.rebuildNameSegmentVocab();
  801. }
  802. } catch { /* vocab is advisory — never fail a sync over it */ }
  803. return result;
  804. } finally {
  805. this.fileLock.release();
  806. }
  807. });
  808. }
  809. /**
  810. * Check if an indexing operation is currently in progress
  811. */
  812. isIndexing(): boolean {
  813. return this.indexMutex.isLocked();
  814. }
  815. // ===========================================================================
  816. // File Watching
  817. // ===========================================================================
  818. /**
  819. * Start watching for file changes and auto-syncing.
  820. *
  821. * Uses native OS file events (FSEvents on macOS, inotify on Linux 19+,
  822. * ReadDirectoryChangesW on Windows) with debouncing to avoid thrashing.
  823. *
  824. * @param options - Watch options (debounce delay, callbacks)
  825. * @returns true if watching started successfully
  826. */
  827. watch(options: WatchOptions = {}): boolean {
  828. if (this.watcher?.isActive()) return true;
  829. this.watcher = new FileWatcher(
  830. this.projectRoot,
  831. async () => {
  832. const result = await this.sync();
  833. // sync() returns this exact zero-shape iff it failed to acquire the
  834. // file lock (a real empty sync always has filesChecked > 0 because
  835. // scanDirectory ran). Surface that to the watcher as a typed error
  836. // so it keeps pendingFiles + reschedules instead of clearing them
  837. // (#449).
  838. if (result.filesChecked === 0 && result.durationMs === 0) {
  839. throw new LockUnavailableError();
  840. }
  841. const filesChanged = result.filesAdded + result.filesModified + result.filesRemoved;
  842. return { filesChanged, durationMs: result.durationMs };
  843. },
  844. options
  845. );
  846. return this.watcher.start();
  847. }
  848. /**
  849. * Stop watching for file changes.
  850. */
  851. unwatch(): void {
  852. if (this.watcher) {
  853. this.watcher.stop();
  854. this.watcher = null;
  855. }
  856. }
  857. /**
  858. * Check if the file watcher is active.
  859. */
  860. isWatching(): boolean {
  861. return this.watcher?.isActive() ?? false;
  862. }
  863. /**
  864. * True once live watching has permanently degraded (OS watch-resource
  865. * exhaustion, or a write lock held past the retry budget) and auto-sync is
  866. * disabled until the next {@link watch} call. Distinct from `!isWatching()`:
  867. * a stopped/never-started watcher is inactive but NOT degraded. MCP tools use
  868. * this to surface a whole-index "results may be stale" notice, since
  869. * `getPendingFiles()` goes empty once watching stops (#876).
  870. */
  871. isWatcherDegraded(): boolean {
  872. return this.watcher?.isDegraded() ?? false;
  873. }
  874. /** The reason live watching degraded, or null if it is healthy (#876). */
  875. getWatcherDegradedReason(): string | null {
  876. return this.watcher?.getDegradedReason() ?? null;
  877. }
  878. /**
  879. * Files seen by the file watcher since the last successful sync —
  880. * the per-file "stale" signal MCP tools attach to responses so an agent
  881. * can fall back to {@link Read} for just the affected file without
  882. * waiting for a debounced sync to complete (issue #403).
  883. *
  884. * Returns an empty list when the watcher isn't active, or no events have
  885. * arrived. Each entry includes `firstSeenMs` and `lastSeenMs` (wall-clock
  886. * `Date.now()` values) so callers can render "edited Nms ago", plus an
  887. * `indexing` flag indicating whether the in-flight sync (if any) will
  888. * absorb that file.
  889. */
  890. getPendingFiles(): PendingFile[] {
  891. return this.watcher?.getPendingFiles() ?? [];
  892. }
  893. /**
  894. * Resolves once the file watcher has installed its watch set. Useful for
  895. * tests that need a deterministic boundary before asserting on
  896. * `getPendingFiles()`. Resolves immediately when no watcher is active.
  897. */
  898. waitUntilWatcherReady(timeoutMs?: number): Promise<void> {
  899. return this.watcher ? this.watcher.waitUntilReady(timeoutMs) : Promise.resolve();
  900. }
  901. /**
  902. * Get files that have changed since last index
  903. */
  904. getChangedFiles(): { added: string[]; modified: string[]; removed: string[] } {
  905. return this.orchestrator.getChangedFiles();
  906. }
  907. /**
  908. * Most recent index timestamp (ms since epoch) across all tracked files, or
  909. * null when nothing is indexed yet. Lets library consumers check index
  910. * freshness without shelling out to `codegraph status --json`. (#329)
  911. */
  912. getLastIndexedAt(): number | null {
  913. return this.queries.getLastIndexedAt();
  914. }
  915. /**
  916. * Completeness of the last full index run. `'complete'` is the only good
  917. * state. `'indexing'` after the fact means a run was killed mid-index (OOM,
  918. * SIGKILL, liveness watchdog) and the on-disk index is truncated;
  919. * `'partial'` means the run finished but silently dropped files
  920. * (discovered > indexed+skipped+errored); `'failed'` means it reported
  921. * failure. `null` = index predates this marker. Surfaced by
  922. * `codegraph status`.
  923. */
  924. getIndexState(): 'indexing' | 'complete' | 'partial' | 'failed' | null {
  925. const raw = this.queries.getMetadata('index_state');
  926. return raw === 'indexing' || raw === 'complete' || raw === 'partial' || raw === 'failed'
  927. ? raw
  928. : null;
  929. }
  930. /**
  931. * Which engine built the current index: the package version + extraction
  932. * version stamped at the last full `indexAll`. Either field is null for an
  933. * index built before stamping existed (treated as stale). See
  934. * `extraction-version.ts` and `isIndexStale()`.
  935. */
  936. getIndexBuildInfo(): { version: string | null; extractionVersion: number | null } {
  937. const version = this.queries.getMetadata('indexed_with_version');
  938. const ev = this.queries.getMetadata('indexed_with_extraction_version');
  939. const parsed = ev != null ? parseInt(ev, 10) : NaN;
  940. return { version, extractionVersion: Number.isFinite(parsed) ? parsed : null };
  941. }
  942. /**
  943. * True when the on-disk index was built by an engine whose extraction is
  944. * older than the one now running — i.e. a re-index would add data a migration
  945. * can't backfill. False when there's no index yet (nothing to refresh) or the
  946. * stamp is current. This is the signal behind `codegraph status`'s re-index
  947. * hint and `codegraph upgrade`'s reminder.
  948. */
  949. isIndexStale(): boolean {
  950. if (this.queries.getLastIndexedAt() == null) return false;
  951. const { extractionVersion } = this.getIndexBuildInfo();
  952. return extractionVersion == null || extractionVersion < EXTRACTION_VERSION;
  953. }
  954. /**
  955. * Extract nodes and edges from source code (without storing)
  956. */
  957. extractFromSource(filePath: string, source: string): ExtractionResult {
  958. return extractFromSource(filePath, source);
  959. }
  960. // ===========================================================================
  961. // Reference Resolution
  962. // ===========================================================================
  963. /**
  964. * Resolve unresolved references and create edges
  965. *
  966. * This method takes unresolved references from extraction and attempts
  967. * to resolve them using multiple strategies:
  968. * - Framework-specific patterns (React, Express, Laravel)
  969. * - Import-based resolution
  970. * - Name-based symbol matching
  971. */
  972. resolveReferences(onProgress?: (current: number, total: number) => void): ResolutionResult {
  973. // Get all unresolved references from the database
  974. const unresolvedRefs = this.queries.getUnresolvedReferences();
  975. return this.resolver.resolveAndPersist(unresolvedRefs, onProgress);
  976. }
  977. /**
  978. * Resolve references in batches to keep memory bounded on large codebases.
  979. * Processes chunks of unresolved refs, persisting results after each batch.
  980. */
  981. async resolveReferencesBatched(
  982. onProgress?: (current: number, total: number) => void,
  983. onSynthesisProgress?: (done: number, total: number) => void
  984. ): Promise<ResolutionResult> {
  985. return this.resolver.resolveAndPersistBatched(onProgress, undefined, onSynthesisProgress, {
  986. dbPath: this.db.getPath(),
  987. });
  988. }
  989. /**
  990. * References extracted but never attempted by a resolution pass. Zero on a
  991. * healthy index — a completed pass consumes every pending row (resolving it
  992. * or parking it as failed, #1240). Non-zero at rest means a pass was
  993. * interrupted mid-run (killed indexer, crash — #1187), so some files' call
  994. * edges are missing; the next `sync` sweeps them.
  995. */
  996. getPendingReferenceCount(): number {
  997. return this.queries.getUnresolvedReferencesCount();
  998. }
  999. /**
  1000. * Get detected frameworks in the project
  1001. */
  1002. getDetectedFrameworks(): string[] {
  1003. return this.resolver.getDetectedFrameworks();
  1004. }
  1005. /**
  1006. * Re-initialize the resolver (useful after adding new files)
  1007. */
  1008. reinitializeResolver(): void {
  1009. this.resolver.initialize();
  1010. }
  1011. // ===========================================================================
  1012. // Graph Statistics
  1013. // ===========================================================================
  1014. /**
  1015. * Get statistics about the knowledge graph
  1016. */
  1017. getStats(): GraphStats {
  1018. const stats = this.queries.getStats();
  1019. stats.dbSizeBytes = this.db.getSize();
  1020. return stats;
  1021. }
  1022. /**
  1023. * Active SQLite backend for this project's connection (`node-sqlite` — Node's
  1024. * built-in real-SQLite module). Surfaced via `codegraph status` and the
  1025. * `codegraph_status` MCP tool alongside the effective journal mode.
  1026. */
  1027. getBackend(): import('./db').SqliteBackend {
  1028. return this.db.getBackend();
  1029. }
  1030. /**
  1031. * The journal mode actually in effect ('wal', 'delete', …). 'wal' means
  1032. * readers never block on a concurrent writer; anything else means they can,
  1033. * which is the precondition for the "database is locked" failures in issue
  1034. * #238. Surfaced via `codegraph status` and the `codegraph_status` MCP tool.
  1035. */
  1036. getJournalMode(): string {
  1037. return this.db.getJournalMode();
  1038. }
  1039. // ===========================================================================
  1040. // Node Operations
  1041. // ===========================================================================
  1042. /**
  1043. * Get a node by ID
  1044. */
  1045. getNode(id: string): Node | null {
  1046. return this.queries.getNodeById(id);
  1047. }
  1048. /**
  1049. * Get all nodes in a file
  1050. */
  1051. getNodesInFile(filePath: string): Node[] {
  1052. return this.queries.getNodesByFile(filePath);
  1053. }
  1054. /**
  1055. * Get all nodes of a specific kind
  1056. */
  1057. getNodesByKind(kind: Node['kind']): Node[] {
  1058. return this.queries.getNodesByKind(kind);
  1059. }
  1060. /**
  1061. * Get ALL nodes with an exact name (direct index lookup, not FTS-ranked/capped).
  1062. * Used to enumerate every overload of a heavily-overloaded name so the specific
  1063. * definition the caller wants is never dropped below a search cut.
  1064. */
  1065. getNodesByName(name: string): Node[] {
  1066. return this.queries.getNodesByName(name);
  1067. }
  1068. /** Nodes whose name starts with `prefix` (index range scan, capped). */
  1069. getNodesByNamePrefix(prefix: string, limit = 20): Node[] {
  1070. return this.queries.getNodesByNamePrefix(prefix, limit);
  1071. }
  1072. /**
  1073. * Search nodes by text
  1074. */
  1075. searchNodes(query: string, options?: SearchOptions): SearchResult[] {
  1076. return this.queries.searchNodes(query, options);
  1077. }
  1078. /**
  1079. * Graph-derived prompt matching for the front-load hook's MEDIUM tier:
  1080. * which indexed symbols do these prose words name? "state machine des
  1081. * commandes" → `OrderStateMachine`, in any human language whose technical
  1082. * nouns are Latin script — no keyword list involved.
  1083. *
  1084. * Precision comes from the repo's own naming statistics, not vocabulary:
  1085. * - CO-OCCURRENCE: ≥2 words that are segments of the SAME name ("state" +
  1086. * "machine" → OrderStateMachine) is strong evidence and always qualifies.
  1087. * - RARITY: a single matched word qualifies only when its segment is
  1088. * discriminative here (≤ {@link SEGMENT_RARITY_CEILING} distinct names) —
  1089. * "checkout" in a shop backend yes, "state" in a react app no.
  1090. * Every candidate is re-verified against `nodes` before being returned
  1091. * (vocab rows are proposals; deletions leave orphans by design), so a
  1092. * returned symbol is guaranteed to exist right now.
  1093. */
  1094. getSegmentMatches(words: string[], limit: number = 6): SegmentMatch[] {
  1095. if (words.length === 0) return [];
  1096. // Variant → original word (plural folding), for coverage accounting.
  1097. const variantToWord = new Map<string, string>();
  1098. for (const word of words) {
  1099. for (const variant of segmentLookupVariants(word)) {
  1100. if (!variantToWord.has(variant)) variantToWord.set(variant, word);
  1101. }
  1102. }
  1103. const variants = [...variantToWord.keys()];
  1104. // Tier A: co-occurrence. The SQL folds variants back to their original
  1105. // word (#1146), so minWords=2 means two distinct PROMPT WORDS — a name
  1106. // matching both `service` and `services` can't tie with (or crowd past
  1107. // the LIMIT) a genuine two-word match. The JS re-check below recomputes
  1108. // the fold from live segments as the honesty layer.
  1109. const variantPairs = [...variantToWord.entries()].map(([segment, word]) => ({ segment, word }));
  1110. const candidates: Array<{ name: string; matchedWords: Set<string> }> = [];
  1111. for (const hit of this.queries.getSegmentCoOccurrence(variantPairs, 2, 24)) {
  1112. const matched = this.wordsMatchingName(hit.name, variantToWord);
  1113. if (matched.size >= 2) candidates.push({ name: hit.name, matchedWords: matched });
  1114. }
  1115. // Tier B: single rare word. Only when co-occurrence found nothing — a
  1116. // co-occurring name is categorically stronger evidence — and under
  1117. // stricter rules, because one word is thin: the word must be ≥5 chars
  1118. // (measured FPs: "this", "typo"); the segment must appear in AT LEAST TWO
  1119. // names (a concept the codebase is about clusters across names —
  1120. // CheckoutService/CheckoutController — while a prose coincidence is a
  1121. // singleton: measured FP "deploy to PRODUCTION" → the one name
  1122. // matchesNonProductionDir); and the candidate name must have ≥2 segments
  1123. // (a bare common verb matching a bare function name — "write" → `write` —
  1124. // is prose coincidence, not the user naming a symbol).
  1125. if (candidates.length === 0) {
  1126. const singleWordVariants = variants.filter((v) => variantToWord.get(v)!.length >= 5);
  1127. const counts = this.queries.getSegmentNameCounts(singleWordVariants);
  1128. const rare = [...counts.entries()]
  1129. .filter(([, n]) => n >= 2 && n <= CodeGraph.SEGMENT_RARITY_CEILING)
  1130. .sort((a, b) => a[1] - b[1])
  1131. .slice(0, 2);
  1132. for (const [variant] of rare) {
  1133. const word = variantToWord.get(variant)!;
  1134. for (const name of this.queries.getNamesForSegment(variant, 12)) {
  1135. if (splitIdentifierSegments(name).length < 2) continue;
  1136. candidates.push({ name, matchedWords: new Set([word]) });
  1137. }
  1138. }
  1139. }
  1140. // Verify against nodes (the honesty gate) and pick a representative
  1141. // definition per name. A name whose only nodes are file/import kind has
  1142. // no real definition to point at — surfacing the import statement instead
  1143. // reads as a matched symbol but isn't one (#1144) — so it's skipped, the
  1144. // same way an orphaned vocab row is. (Import names no longer enter the
  1145. // vocab at write time, but rows written before that exclusion persist
  1146. // until the next full index.)
  1147. const out: SegmentMatch[] = [];
  1148. const seen = new Set<string>();
  1149. candidates.sort((a, b) => b.matchedWords.size - a.matchedWords.size || a.name.length - b.name.length);
  1150. for (const candidate of candidates) {
  1151. if (out.length >= limit) break;
  1152. if (seen.has(candidate.name)) continue;
  1153. seen.add(candidate.name);
  1154. const nodes = this.queries.getNodesByName(candidate.name);
  1155. if (nodes.length === 0) continue; // orphaned vocab row — name no longer exists
  1156. const rep = nodes.find((n) => n.kind !== 'file' && n.kind !== 'import');
  1157. if (!rep) continue; // no real definition — don't surface an import/file as one
  1158. out.push({
  1159. name: candidate.name,
  1160. kind: rep.kind,
  1161. filePath: rep.filePath,
  1162. startLine: rep.startLine ?? 0,
  1163. matchedWords: [...candidate.matchedWords].sort(),
  1164. });
  1165. }
  1166. return out;
  1167. }
  1168. /** A single word ("state") can match hundreds of names in a big repo — that
  1169. * is noise, not signal. Ceiling for the single-word tier; co-occurrence is
  1170. * exempt because two words on one name is already discriminative. */
  1171. private static readonly SEGMENT_RARITY_CEILING = 25;
  1172. /** Which of the prompt's original words match `name`'s segments (via
  1173. * variants). Segments are recomputed in JS — a name-keyed vocab lookup
  1174. * would scan the (segment, name) primary key. */
  1175. private wordsMatchingName(name: string, variantToWord: Map<string, string>): Set<string> {
  1176. const segments = new Set(splitIdentifierSegments(name));
  1177. const matched = new Set<string>();
  1178. for (const [variant, word] of variantToWord) {
  1179. if (segments.has(variant)) matched.add(word);
  1180. }
  1181. return matched;
  1182. }
  1183. /**
  1184. * One-shot upgrade heal for callers that open the graph WITHOUT syncing —
  1185. * concretely the prompt hook, whose MEDIUM tier reads the segment
  1186. * vocabulary: a database migrated from before the vocab table existed
  1187. * starts with it empty, and the only other backfill lives inside `sync()`,
  1188. * which such callers never run (#1142). Returns true when the vocab is
  1189. * usable (already populated — the overwhelmingly common one-SELECT case —
  1190. * or healed here); false when it isn't (empty graph, or another process
  1191. * holds the index lock — that process's own sync heals it).
  1192. */
  1193. async healSegmentVocabIfEmpty(): Promise<boolean> {
  1194. const empty = (() => {
  1195. try { return this.queries.isNameSegmentVocabEmpty(); } catch { return false; }
  1196. })();
  1197. if (!empty) return true;
  1198. if (this.queries.getNodeAndEdgeCount().nodes === 0) return false;
  1199. return this.indexMutex.withLock(async () => {
  1200. try {
  1201. this.fileLock.acquire();
  1202. } catch {
  1203. return false; // an index/sync is running — it backfills the vocab itself
  1204. }
  1205. try {
  1206. if (!this.queries.isNameSegmentVocabEmpty()) return true; // raced: healed meanwhile
  1207. await this.rebuildNameSegmentVocab();
  1208. return true;
  1209. } finally {
  1210. this.fileLock.release();
  1211. }
  1212. });
  1213. }
  1214. /**
  1215. * Rebuild the segment vocabulary from the current graph, batched and
  1216. * yielding — the upgrade-heal path for indexes built before the vocab table
  1217. * existed. Runs inside the index mutex/lock (sync and
  1218. * healSegmentVocabIfEmpty hold them).
  1219. */
  1220. private async rebuildNameSegmentVocab(): Promise<void> {
  1221. const maybeYield = createYielder();
  1222. const BATCH = 2000;
  1223. for (let offset = 0; ; offset += BATCH) {
  1224. const names = this.queries.getDistinctNodeNames(BATCH, offset);
  1225. if (names.length === 0) break;
  1226. this.queries.insertNameSegmentsBatch(names);
  1227. await maybeYield();
  1228. }
  1229. }
  1230. /**
  1231. * Normalized project-name tokens (go.mod / package.json / repo dir) used to
  1232. * down-weight the non-discriminative project name in search ranking (#720).
  1233. * Exposed so explore can exclude it from the PascalCase type-disambiguation
  1234. * bias, which would otherwise pull overloaded tokens toward whichever stack
  1235. * embeds the project name.
  1236. */
  1237. getProjectNameTokens(): Set<string> {
  1238. return this.queries.getProjectNameTokens();
  1239. }
  1240. /**
  1241. * Find the project's "primary route file" — the file with the densest
  1242. * concentration of framework-emitted `route` nodes (≥3 routes, ≥30%
  1243. * of all non-test routes). Used to inline the routing config in
  1244. * `codegraph_explore` responses on small realworld template repos
  1245. * (rails-realworld, laravel-realworld, drupal-admintoolbar, …) where
  1246. * Glob+Read of `routes.rb`/`urls.py`/etc. otherwise beats codegraph.
  1247. */
  1248. getTopRouteFile(): { filePath: string; routeCount: number; totalRoutes: number } | null {
  1249. return this.queries.getTopRouteFile();
  1250. }
  1251. /**
  1252. * Build a URL → handler routing manifest from the index. Each entry
  1253. * pairs a route node (URL + method) with its handler function/method
  1254. * via the `references` edge that framework resolvers emit. Returns
  1255. * null when fewer than 3 valid (non-test) routes exist.
  1256. */
  1257. getRoutingManifest(limit?: number): {
  1258. entries: Array<{ url: string; handler: string; handlerFile: string; handlerLine: number; handlerKind: string }>;
  1259. topHandlerFile: string | null;
  1260. topHandlerFileCount: number;
  1261. totalRoutes: number;
  1262. } | null {
  1263. return this.queries.getRoutingManifest(limit);
  1264. }
  1265. // ===========================================================================
  1266. // Edge Operations
  1267. // ===========================================================================
  1268. /**
  1269. * Get outgoing edges from a node
  1270. */
  1271. getOutgoingEdges(nodeId: string): Edge[] {
  1272. return this.queries.getOutgoingEdges(nodeId);
  1273. }
  1274. /**
  1275. * Get incoming edges to a node
  1276. */
  1277. getIncomingEdges(nodeId: string): Edge[] {
  1278. return this.queries.getIncomingEdges(nodeId);
  1279. }
  1280. // ===========================================================================
  1281. // File Operations
  1282. // ===========================================================================
  1283. /**
  1284. * Get a file record by path
  1285. */
  1286. getFile(filePath: string): FileRecord | null {
  1287. return this.queries.getFileByPath(filePath);
  1288. }
  1289. /**
  1290. * Get all tracked files
  1291. */
  1292. getFiles(): FileRecord[] {
  1293. return this.queries.getAllFiles();
  1294. }
  1295. // ===========================================================================
  1296. // Graph Query Methods
  1297. // ===========================================================================
  1298. /**
  1299. * Get the context for a node (ancestors, children, references)
  1300. *
  1301. * Returns comprehensive context about a node including its containment
  1302. * hierarchy, children, incoming/outgoing references, type information,
  1303. * and relevant imports.
  1304. *
  1305. * @param nodeId - ID of the focal node
  1306. * @returns Context object with all related information
  1307. */
  1308. getContext(nodeId: string): Context {
  1309. return this.graphManager.getContext(nodeId);
  1310. }
  1311. /**
  1312. * Traverse the graph from a starting node
  1313. *
  1314. * Uses breadth-first search by default. Supports filtering by edge types,
  1315. * node types, and traversal direction.
  1316. *
  1317. * @param startId - Starting node ID
  1318. * @param options - Traversal options
  1319. * @returns Subgraph containing traversed nodes and edges
  1320. */
  1321. traverse(startId: string, options?: TraversalOptions): Subgraph {
  1322. return this.traverser.traverseBFS(startId, options);
  1323. }
  1324. /**
  1325. * Get the call graph for a function
  1326. *
  1327. * Returns both callers (functions that call this function) and
  1328. * callees (functions called by this function) up to the specified depth.
  1329. *
  1330. * @param nodeId - ID of the function/method node
  1331. * @param depth - Maximum depth in each direction (default: 2)
  1332. * @returns Subgraph containing the call graph
  1333. */
  1334. getCallGraph(nodeId: string, depth: number = 2): Subgraph {
  1335. return this.traverser.getCallGraph(nodeId, depth);
  1336. }
  1337. /**
  1338. * Get the type hierarchy for a class/interface
  1339. *
  1340. * Returns both ancestors (types this extends/implements) and
  1341. * descendants (types that extend/implement this).
  1342. *
  1343. * @param nodeId - ID of the class/interface node
  1344. * @returns Subgraph containing the type hierarchy
  1345. */
  1346. getTypeHierarchy(nodeId: string): Subgraph {
  1347. return this.traverser.getTypeHierarchy(nodeId);
  1348. }
  1349. /**
  1350. * Find all usages of a symbol
  1351. *
  1352. * Returns all nodes that reference the specified symbol through
  1353. * any edge type (calls, references, type_of, etc.).
  1354. *
  1355. * @param nodeId - ID of the symbol node
  1356. * @returns Array of nodes and edges that reference this symbol
  1357. */
  1358. findUsages(nodeId: string): Array<{ node: Node; edge: Edge }> {
  1359. return this.traverser.findUsages(nodeId);
  1360. }
  1361. /**
  1362. * Get callers of a function/method
  1363. *
  1364. * @param nodeId - ID of the function/method node
  1365. * @param maxDepth - Maximum depth to traverse (default: 1)
  1366. * @returns Array of nodes that call this function
  1367. */
  1368. getCallers(nodeId: string, maxDepth: number = 1): Array<{ node: Node; edge: Edge }> {
  1369. return this.traverser.getCallers(nodeId, maxDepth);
  1370. }
  1371. /**
  1372. * Get callees of a function/method
  1373. *
  1374. * @param nodeId - ID of the function/method node
  1375. * @param maxDepth - Maximum depth to traverse (default: 1)
  1376. * @returns Array of nodes called by this function
  1377. */
  1378. getCallees(nodeId: string, maxDepth: number = 1): Array<{ node: Node; edge: Edge }> {
  1379. return this.traverser.getCallees(nodeId, maxDepth);
  1380. }
  1381. /**
  1382. * Calculate the impact radius of a node
  1383. *
  1384. * Returns all nodes that could be affected by changes to this node.
  1385. *
  1386. * @param nodeId - ID of the node
  1387. * @param maxDepth - Maximum depth to traverse (default: 3)
  1388. * @returns Subgraph containing potentially impacted nodes
  1389. */
  1390. getImpactRadius(nodeId: string, maxDepth: number = 3): Subgraph {
  1391. return this.traverser.getImpactRadius(nodeId, maxDepth);
  1392. }
  1393. /**
  1394. * Find the shortest path between two nodes
  1395. *
  1396. * @param fromId - Starting node ID
  1397. * @param toId - Target node ID
  1398. * @param edgeKinds - Edge types to consider (all if empty)
  1399. * @returns Array of nodes and edges forming the path, or null if no path exists
  1400. */
  1401. findPath(
  1402. fromId: string,
  1403. toId: string,
  1404. edgeKinds?: Edge['kind'][]
  1405. ): Array<{ node: Node; edge: Edge | null }> | null {
  1406. return this.traverser.findPath(fromId, toId, edgeKinds);
  1407. }
  1408. /**
  1409. * Get ancestors of a node in the containment hierarchy
  1410. *
  1411. * @param nodeId - ID of the node
  1412. * @returns Array of ancestor nodes from immediate parent to root
  1413. */
  1414. getAncestors(nodeId: string): Node[] {
  1415. return this.traverser.getAncestors(nodeId);
  1416. }
  1417. /**
  1418. * Get immediate children of a node
  1419. *
  1420. * @param nodeId - ID of the node
  1421. * @returns Array of child nodes
  1422. */
  1423. getChildren(nodeId: string): Node[] {
  1424. return this.traverser.getChildren(nodeId);
  1425. }
  1426. /**
  1427. * Get dependencies of a file
  1428. *
  1429. * @param filePath - Path to the file
  1430. * @returns Array of file paths this file depends on
  1431. */
  1432. getFileDependencies(filePath: string): string[] {
  1433. return this.graphManager.getFileDependencies(filePath);
  1434. }
  1435. /**
  1436. * Get dependents of a file
  1437. *
  1438. * @param filePath - Path to the file
  1439. * @returns Array of file paths that depend on this file
  1440. */
  1441. getFileDependents(filePath: string): string[] {
  1442. return this.graphManager.getFileDependents(filePath);
  1443. }
  1444. /**
  1445. * Find circular dependencies in the codebase
  1446. *
  1447. * @returns Array of cycles, each cycle is an array of file paths
  1448. */
  1449. findCircularDependencies(): string[][] {
  1450. return this.graphManager.findCircularDependencies();
  1451. }
  1452. /**
  1453. * Find dead code (unreferenced symbols)
  1454. *
  1455. * @param kinds - Node kinds to check (default: functions, methods, classes)
  1456. * @returns Array of unreferenced nodes
  1457. */
  1458. findDeadCode(kinds?: Node['kind'][]): Node[] {
  1459. return this.graphManager.findDeadCode(kinds);
  1460. }
  1461. /**
  1462. * Get complexity metrics for a node
  1463. *
  1464. * @param nodeId - ID of the node
  1465. * @returns Object containing various complexity metrics
  1466. */
  1467. getNodeMetrics(nodeId: string): {
  1468. incomingEdgeCount: number;
  1469. outgoingEdgeCount: number;
  1470. callCount: number;
  1471. callerCount: number;
  1472. childCount: number;
  1473. depth: number;
  1474. } {
  1475. return this.graphManager.getNodeMetrics(nodeId);
  1476. }
  1477. // ===========================================================================
  1478. // Context Building
  1479. // ===========================================================================
  1480. /**
  1481. * Get the source code for a node
  1482. *
  1483. * Reads the file and extracts the code between startLine and endLine.
  1484. *
  1485. * @param nodeId - ID of the node
  1486. * @returns Code string or null if not found
  1487. */
  1488. async getCode(nodeId: string): Promise<string | null> {
  1489. return this.contextBuilder.getCode(nodeId);
  1490. }
  1491. /**
  1492. * Find relevant subgraph for a query
  1493. *
  1494. * Combines semantic search with graph traversal to find the most
  1495. * relevant nodes and their relationships for a given query.
  1496. *
  1497. * @param query - Natural language query describing the task
  1498. * @param options - Search and traversal options
  1499. * @returns Subgraph of relevant nodes and edges
  1500. */
  1501. async findRelevantContext(
  1502. query: string,
  1503. options?: FindRelevantContextOptions
  1504. ): Promise<Subgraph> {
  1505. return this.contextBuilder.findRelevantContext(query, options);
  1506. }
  1507. /**
  1508. * Build context for a task
  1509. *
  1510. * Creates comprehensive context by:
  1511. * 1. Running FTS search to find entry points
  1512. * 2. Expanding the graph around entry points
  1513. * 3. Extracting code blocks for key nodes
  1514. * 4. Formatting output for Claude
  1515. *
  1516. * @param input - Task description (string or {title, description})
  1517. * @param options - Build options (maxNodes, includeCode, format, etc.)
  1518. * @returns TaskContext object or formatted string (markdown/JSON)
  1519. */
  1520. async buildContext(
  1521. input: TaskInput,
  1522. options?: BuildContextOptions
  1523. ): Promise<TaskContext | string> {
  1524. return this.contextBuilder.buildContext(input, options);
  1525. }
  1526. // ===========================================================================
  1527. // Database Management
  1528. // ===========================================================================
  1529. /**
  1530. * Optimize the database (vacuum and analyze)
  1531. */
  1532. optimize(): void {
  1533. this.db.optimize();
  1534. }
  1535. /**
  1536. * Clear all data from the graph
  1537. */
  1538. clear(): void {
  1539. this.queries.clear();
  1540. }
  1541. /**
  1542. * Alias for close() for backwards compatibility.
  1543. * @deprecated Use close() instead
  1544. */
  1545. destroy(): void {
  1546. this.close();
  1547. }
  1548. /**
  1549. * Completely remove CodeGraph from the project.
  1550. * This closes the database and deletes the .CodeGraph directory.
  1551. *
  1552. * WARNING: This permanently deletes all CodeGraph data for the project.
  1553. */
  1554. uninitialize(): void {
  1555. this.close();
  1556. removeDirectory(this.projectRoot);
  1557. }
  1558. }
  1559. // Default export
  1560. export default CodeGraph;