index.ts 84 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110
  1. /**
  2. * CodeGraph
  3. *
  4. * A local-first code intelligence system that builds a semantic
  5. * knowledge graph from any codebase.
  6. */
  7. import * as path from 'path';
  8. import {
  9. Node,
  10. NodeKind,
  11. Edge,
  12. FileRecord,
  13. ExtractionResult,
  14. Subgraph,
  15. TraversalOptions,
  16. SearchOptions,
  17. SearchResult,
  18. SegmentMatch,
  19. Context,
  20. GraphStats,
  21. TaskInput,
  22. TaskContext,
  23. BuildContextOptions,
  24. FindRelevantContextOptions,
  25. UnresolvedReference,
  26. } from './types';
  27. import { DatabaseConnection, getDatabasePath, removeDatabaseFiles } from './db';
  28. import { WalCheckpointValve, resolveWalValveMb } from './db/wal-valve';
  29. import { QueryBuilder } from './db/queries';
  30. import {
  31. isInitialized,
  32. createDirectory,
  33. removeDirectory,
  34. validateDirectory,
  35. } from './directory';
  36. import {
  37. ExtractionOrchestrator,
  38. IndexProgress,
  39. IndexResult,
  40. SyncResult,
  41. extractFromSource,
  42. initGrammars,
  43. } from './extraction';
  44. import {
  45. ReferenceResolver,
  46. createResolver,
  47. ResolutionResult,
  48. } from './resolution';
  49. import { GraphTraverser, GraphQueryManager } from './graph';
  50. import { ContextBuilder, createContextBuilder } from './context';
  51. import { Mutex, FileLock } from './utils';
  52. import { FileWatcher, WatchOptions, PendingFile, LockUnavailableError } from './sync';
  53. import { EXTRACTION_VERSION } from './extraction/extraction-version';
  54. import { getCodeGraphDir } from './directory';
  55. import { deriveProjectNameTokens } from './search/query-utils';
  56. import ignore from 'ignore';
  57. import { loadDeprioritizePatterns } from './project-config';
  58. import { CodeGraphPackageVersion } from './mcp/version';
  59. import { extractSegmentSearchWords, segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments';
  60. import { createYielder } from './resolution/cooperative-yield';
  61. import { minRefsForPool } from './resolution/resolver-pool';
  62. // Re-export types for consumers
  63. export * from './types';
  64. // Storage building blocks for embedded/SDK consumers that drive the graph
  65. // directly (open a DB, run prepared queries) rather than through the CodeGraph
  66. // facade. Exposed from the package entry so they no longer require deep imports
  67. // into dist/ (issue #354).
  68. export { getDatabasePath, DatabaseConnection } from './db';
  69. export { QueryBuilder } from './db/queries';
  70. export {
  71. getCodeGraphDir,
  72. isInitialized,
  73. findNearestCodeGraphRoot,
  74. CODEGRAPH_DIR,
  75. } from './directory';
  76. export { IndexProgress, IndexResult, SyncResult } from './extraction';
  77. export { detectLanguage, isLanguageSupported, isGrammarLoaded, getSupportedLanguages, initGrammars, loadGrammarsForLanguages, loadAllGrammars } from './extraction';
  78. export { ResolutionResult } from './resolution';
  79. export {
  80. CodeGraphError,
  81. FileError,
  82. ParseError,
  83. DatabaseError,
  84. SearchError,
  85. VectorError,
  86. ConfigError,
  87. Logger,
  88. setLogger,
  89. getLogger,
  90. silentLogger,
  91. defaultLogger,
  92. } from './errors';
  93. export { Mutex, FileLock, processInBatches, debounce, throttle, MemoryMonitor } from './utils';
  94. export { FileWatcher, WatchOptions, PendingFile, LockUnavailableError } from './sync';
  95. export { MCPServer } from './mcp';
  96. /**
  97. * Options for initializing a new CodeGraph project
  98. */
  99. export interface InitOptions {
  100. /** Whether to run initial indexing after init */
  101. index?: boolean;
  102. /** Progress callback for indexing */
  103. onProgress?: (progress: IndexProgress) => void;
  104. }
  105. /**
  106. * Options for opening an existing CodeGraph project
  107. */
  108. export interface OpenOptions {
  109. /** Whether to run sync if files have changed */
  110. sync?: boolean;
  111. /** Whether to run in read-only mode */
  112. readOnly?: boolean;
  113. }
  114. /**
  115. * Options for indexing
  116. */
  117. export interface IndexOptions {
  118. /** Progress callback */
  119. onProgress?: (progress: IndexProgress) => void;
  120. /** Abort signal for cancellation */
  121. signal?: AbortSignal;
  122. /** Enable verbose logging (worker lifecycle, memory, timeouts) */
  123. verbose?: boolean;
  124. /** Watcher fast path: reconcile ONLY these project-relative paths (see ExtractionOrchestrator.sync). */
  125. paths?: string[];
  126. }
  127. /**
  128. * Main CodeGraph class
  129. *
  130. * Provides the primary interface for interacting with the code knowledge graph.
  131. */
  132. export class CodeGraph {
  133. private db: DatabaseConnection;
  134. private queries: QueryBuilder;
  135. private projectRoot: string;
  136. // Assigned via wireLayers() from the constructor (and again on reopen) — the
  137. // `!` tells TS these are definitely set even though the assignment is one
  138. // method call away from the constructor body.
  139. private orchestrator!: ExtractionOrchestrator;
  140. private resolver!: ReferenceResolver;
  141. private graphManager!: GraphQueryManager;
  142. private traverser!: GraphTraverser;
  143. private contextBuilder!: ContextBuilder;
  144. // Mutex for preventing concurrent indexing operations (in-process)
  145. private indexMutex = new Mutex();
  146. // File lock for preventing concurrent writes across processes (CLI, MCP, git hooks)
  147. private fileLock: FileLock;
  148. // File watcher for auto-sync on file changes
  149. private watcher: FileWatcher | null = null;
  150. private constructor(
  151. db: DatabaseConnection,
  152. queries: QueryBuilder,
  153. projectRoot: string
  154. ) {
  155. this.db = db;
  156. this.queries = queries;
  157. this.projectRoot = projectRoot;
  158. this.fileLock = new FileLock(
  159. path.join(getCodeGraphDir(projectRoot), 'codegraph.lock')
  160. );
  161. this.wireLayers();
  162. }
  163. /**
  164. * (Re)build the query/extraction/graph layers over the current `this.queries`
  165. * (which wraps `this.db`). Factored out of the constructor so `reopenIfReplaced`
  166. * can rebuild them against a fresh connection without duplicating the wiring.
  167. * The path-based `fileLock` is independent of the DB handle, so it stays put.
  168. */
  169. private wireLayers(): void {
  170. // Down-weight the project name as a query term in search ranking — it names
  171. // the whole repo, not a symbol, so it has no discriminative value (#720).
  172. try {
  173. this.queries.setProjectNameTokens(deriveProjectNameTokens(this.projectRoot));
  174. } catch {
  175. // Best-effort: ranking still works without it.
  176. }
  177. // Down-weight the peripheral trees the project named in `codegraph.json`
  178. // `deprioritize` — indexed and findable, but never outranking real code
  179. // (#982). Ranking-only, so a bad pattern costs relevance, never recall.
  180. //
  181. // Read LAZILY, not once here: `wireLayers` runs from the constructor and
  182. // from `reopenIfReplaced`, so a matcher built here would freeze at whatever
  183. // the config said when the project opened. The MCP server caches one
  184. // CodeGraph per root for its whole lifetime, so editing `codegraph.json`
  185. // would appear to do nothing until the process restarted — `exclude` and
  186. // `include` do not behave that way. `loadDeprioritizePatterns` is
  187. // mtime-cached, so this costs one `stat`; the compiled matcher is memoized
  188. // on the pattern array's identity, which the cache keeps stable.
  189. let cachedPatterns: string[] | undefined;
  190. let cachedMatcher: ReturnType<typeof ignore> | undefined;
  191. this.queries.setDeprioritizedPathMatcher((filePath: string): boolean => {
  192. try {
  193. const patterns = loadDeprioritizePatterns(this.projectRoot);
  194. if (patterns.length === 0) return false;
  195. if (patterns !== cachedPatterns) {
  196. cachedPatterns = patterns;
  197. cachedMatcher = ignore().add(patterns);
  198. }
  199. const rel = path.isAbsolute(filePath)
  200. ? path.relative(this.projectRoot, filePath)
  201. : filePath;
  202. if (!rel || rel.startsWith('..')) return false;
  203. return cachedMatcher!.ignores(rel.split(path.sep).join('/'));
  204. } catch {
  205. // Ranking must never take the search down with it.
  206. return false;
  207. }
  208. });
  209. this.orchestrator = new ExtractionOrchestrator(this.projectRoot, this.queries);
  210. this.resolver = createResolver(this.projectRoot, this.queries);
  211. this.graphManager = new GraphQueryManager(this.queries);
  212. this.traverser = new GraphTraverser(this.queries);
  213. this.contextBuilder = createContextBuilder(
  214. this.projectRoot,
  215. this.queries,
  216. this.traverser
  217. );
  218. }
  219. /**
  220. * Heal a stale database handle in place. If `.codegraph/` was removed and
  221. * recreated at the SAME path while this instance held the DB open — a git
  222. * worktree removed and re-added, or `rm -rf .codegraph` + `codegraph init` —
  223. * our open fd points at the now-unlinked inode and can never see the new
  224. * index, so every query returns the pre-removal snapshot until the process
  225. * restarts (#925). When that's detected, open the live file at the same path,
  226. * rebuild the query layers, and swap them IN PLACE, so every holder of this
  227. * instance (the MCP daemon's default project, cached projectPath connections)
  228. * heals without a restart. Returns true iff it reopened.
  229. *
  230. * POSIX-only in practice: `isReplacedOnDisk` never fires on Windows (an open
  231. * file can't be unlinked there, and st_ino is unreliable).
  232. */
  233. reopenIfReplaced(): boolean {
  234. if (!this.db.isReplacedOnDisk()) return false;
  235. const dbPath = this.db.getPath();
  236. // Open the live file FIRST — if that throws (e.g. mid-recreate), the old
  237. // handle stays in place and the caller retries on the next query, rather
  238. // than leaving this instance with no connection at all.
  239. const fresh = DatabaseConnection.open(dbPath);
  240. const stale = this.db;
  241. this.db = fresh;
  242. this.queries = new QueryBuilder(fresh.getDb());
  243. this.wireLayers();
  244. // Releasing the dead handle also frees the leaked db/-wal/-shm fds that were
  245. // pinning the unlinked inode (#925).
  246. try { stale.close(); } catch { /* the old inode is gone; closing just frees fds */ }
  247. return true;
  248. }
  249. // ===========================================================================
  250. // Lifecycle Methods
  251. // ===========================================================================
  252. /**
  253. * Initialize a new CodeGraph project
  254. *
  255. * Creates the .CodeGraph directory, database, and configuration.
  256. *
  257. * @param projectRoot - Path to the project root directory
  258. * @param options - Initialization options
  259. * @returns A new CodeGraph instance
  260. */
  261. static async init(projectRoot: string, options: InitOptions = {}): Promise<CodeGraph> {
  262. await initGrammars();
  263. const resolvedRoot = path.resolve(projectRoot);
  264. // Check if already initialized
  265. if (isInitialized(resolvedRoot)) {
  266. throw new Error(`CodeGraph already initialized in ${resolvedRoot}`);
  267. }
  268. // Create directory structure
  269. createDirectory(resolvedRoot);
  270. // Initialize database
  271. const dbPath = getDatabasePath(resolvedRoot);
  272. const db = DatabaseConnection.initialize(dbPath);
  273. const queries = new QueryBuilder(db.getDb());
  274. const instance = new CodeGraph(db, queries, resolvedRoot);
  275. // Run initial indexing if requested
  276. if (options.index) {
  277. await instance.indexAll({ onProgress: options.onProgress });
  278. }
  279. return instance;
  280. }
  281. /**
  282. * Initialize synchronously (without indexing)
  283. */
  284. static initSync(projectRoot: string): CodeGraph {
  285. const resolvedRoot = path.resolve(projectRoot);
  286. // Check if already initialized
  287. if (isInitialized(resolvedRoot)) {
  288. throw new Error(`CodeGraph already initialized in ${resolvedRoot}`);
  289. }
  290. // Create directory structure
  291. createDirectory(resolvedRoot);
  292. // Initialize database
  293. const dbPath = getDatabasePath(resolvedRoot);
  294. const db = DatabaseConnection.initialize(dbPath);
  295. const queries = new QueryBuilder(db.getDb());
  296. return new CodeGraph(db, queries, resolvedRoot);
  297. }
  298. /**
  299. * Open an existing CodeGraph project
  300. *
  301. * @param projectRoot - Path to the project root directory
  302. * @param options - Open options
  303. * @returns A CodeGraph instance
  304. */
  305. static async open(projectRoot: string, options: OpenOptions = {}): Promise<CodeGraph> {
  306. await initGrammars();
  307. const resolvedRoot = path.resolve(projectRoot);
  308. // Check if initialized
  309. if (!isInitialized(resolvedRoot)) {
  310. throw new Error(`CodeGraph not initialized in ${resolvedRoot}. Run init() first.`);
  311. }
  312. // Validate directory structure
  313. const validation = validateDirectory(resolvedRoot);
  314. if (!validation.valid) {
  315. throw new Error(`Invalid CodeGraph directory: ${validation.errors.join(', ')}`);
  316. }
  317. // Open database
  318. const dbPath = getDatabasePath(resolvedRoot);
  319. const db = DatabaseConnection.open(dbPath);
  320. const queries = new QueryBuilder(db.getDb());
  321. const instance = new CodeGraph(db, queries, resolvedRoot);
  322. // Sync if requested
  323. if (options.sync) {
  324. await instance.sync();
  325. }
  326. return instance;
  327. }
  328. /**
  329. * Rebuild the project's database from scratch and return a fresh, empty
  330. * instance — the "same result as a fresh init" semantics that `codegraph
  331. * index` documents.
  332. *
  333. * Unlike `open()` followed by `clear()`, this DISCARDS the existing
  334. * `.codegraph/codegraph.db` (and its `-wal`/`-shm` sidecars) before
  335. * re-initializing, instead of opening the old database and DELETE-ing every
  336. * row. On a large or pre-fix poisoned index — e.g. an old graph that scanned
  337. * an ignored gitlink corpus (#1065) into ~1.6M nodes with a multi-GB WAL —
  338. * the per-row `nodes_fts` delete-trigger churn blocks the main thread long
  339. * enough to trip the #850 liveness watchdog before indexing even starts, so a
  340. * full re-index could never recover the bad state (#1067). Discarding the
  341. * files is O(1) regardless of size, reclaims the disk, and sidesteps opening
  342. * (and running migrations against) the poisoned database entirely.
  343. */
  344. static async recreate(projectRoot: string): Promise<CodeGraph> {
  345. await initGrammars();
  346. const resolvedRoot = path.resolve(projectRoot);
  347. // Check if initialized — recreate REBUILDS an existing project; it is not a
  348. // first-time `init`.
  349. if (!isInitialized(resolvedRoot)) {
  350. throw new Error(`CodeGraph not initialized in ${resolvedRoot}. Run init() first.`);
  351. }
  352. const dbPath = getDatabasePath(resolvedRoot);
  353. try {
  354. removeDatabaseFiles(dbPath);
  355. } catch (err) {
  356. // POSIX unlinks an open file fine; this fires mainly on Windows when a
  357. // live daemon/MCP server still holds the database. Turn the raw EBUSY into
  358. // an actionable instruction instead of a generic failure.
  359. const reason = err instanceof Error ? err.message : String(err);
  360. throw new Error(
  361. `Could not rebuild the index — the database file is in use (${reason}). ` +
  362. `Stop any running CodeGraph MCP server/daemon for this project and retry, ` +
  363. `or remove the ${getCodeGraphDir(resolvedRoot)} directory and run "codegraph init".`
  364. );
  365. }
  366. // Re-create an empty, freshly-schema'd database at the same path.
  367. const db = DatabaseConnection.initialize(dbPath);
  368. const queries = new QueryBuilder(db.getDb());
  369. return new CodeGraph(db, queries, resolvedRoot);
  370. }
  371. /**
  372. * Open synchronously (without sync)
  373. */
  374. static openSync(projectRoot: string): CodeGraph {
  375. const resolvedRoot = path.resolve(projectRoot);
  376. // Check if initialized
  377. if (!isInitialized(resolvedRoot)) {
  378. throw new Error(`CodeGraph not initialized in ${resolvedRoot}. Run init() first.`);
  379. }
  380. // Validate directory structure
  381. const validation = validateDirectory(resolvedRoot);
  382. if (!validation.valid) {
  383. throw new Error(`Invalid CodeGraph directory: ${validation.errors.join(', ')}`);
  384. }
  385. // Open database
  386. const dbPath = getDatabasePath(resolvedRoot);
  387. const db = DatabaseConnection.open(dbPath);
  388. const queries = new QueryBuilder(db.getDb());
  389. return new CodeGraph(db, queries, resolvedRoot);
  390. }
  391. /**
  392. * Check if a directory has been initialized as a CodeGraph project
  393. */
  394. static isInitialized(projectRoot: string): boolean {
  395. return isInitialized(path.resolve(projectRoot));
  396. }
  397. /**
  398. * Close the CodeGraph instance and release resources
  399. */
  400. close(): void {
  401. this.unwatch();
  402. // Release file lock if held
  403. this.fileLock.release();
  404. this.db.close();
  405. }
  406. /**
  407. * Get the project root directory
  408. */
  409. getProjectRoot(): string {
  410. return this.projectRoot;
  411. }
  412. // ===========================================================================
  413. // Indexing
  414. // ===========================================================================
  415. /**
  416. * Index all files in the project
  417. *
  418. * Uses a mutex to prevent concurrent indexing operations.
  419. */
  420. async indexAll(options: IndexOptions = {}): Promise<IndexResult> {
  421. return this.indexMutex.withLock(async () => {
  422. try {
  423. this.fileLock.acquire();
  424. } catch {
  425. return { success: false, filesIndexed: 0, filesSkipped: 0, filesErrored: 0, nodesCreated: 0, edgesCreated: 0, errors: [{ message: 'Could not acquire file lock - another process may be indexing', severity: 'error' as const }], durationMs: 0 };
  426. }
  427. // Defer WAL auto-checkpointing for the whole bulk run (#1231): the
  428. // default 1000-page interval re-writes hot pages into the main DB file
  429. // over and over — ~95% of all disk I/O during a bulk index, and a
  430. // 19+min → 45s difference on HDD-class storage. The valve bounds WAL
  431. // growth by backfilling PASSIVEly on a worker thread (never blocking
  432. // the writer or the #850 watchdog heartbeat); runMaintenance below does
  433. // the final fold-up before the interval is restored in the finally.
  434. // Kill switch: CODEGRAPH_NO_WAL_DEFER=1. Non-WAL journal modes (some
  435. // network filesystems) have no WAL to defer — skip.
  436. // Fast-init: on a COMPLETELY fresh DB, trade crash-durability for speed
  437. // during the bulk build (journal in memory, no fsync). Safe because the
  438. // DB is disposable until the index completes — index_state stays
  439. // 'indexing' and a crashed init is re-run from scratch; existing DBs
  440. // (re-index/sync) never take this path. Kill switch:
  441. // CODEGRAPH_NO_FAST_INIT=1 (same pattern as CODEGRAPH_NO_WAL_DEFER).
  442. const freshDb = this.queries.getNodeAndEdgeCount().nodes === 0;
  443. const fastInit = process.env.CODEGRAPH_NO_FAST_INIT !== '1' && freshDb;
  444. if (fastInit) {
  445. try {
  446. this.db.getDb().pragma('journal_mode = MEMORY');
  447. this.db.getDb().pragma('synchronous = OFF');
  448. } catch { /* keep WAL */ }
  449. }
  450. const deferWal = !fastInit && process.env.CODEGRAPH_NO_WAL_DEFER !== '1' && this.db.getJournalMode() === 'wal';
  451. let walValve: WalCheckpointValve | null = null;
  452. let priorAutocheckpoint = 1000;
  453. // Set when the fastInit+pool path below defers autocheckpointing, so the
  454. // finally knows to restore the interval on that path too.
  455. let restoreAutocheckpoint = false;
  456. if (deferWal) {
  457. priorAutocheckpoint = this.db.getWalAutocheckpoint();
  458. this.db.setWalAutocheckpoint(0);
  459. walValve = new WalCheckpointValve(
  460. this.db,
  461. resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB, this.db.getDbFileSizeBytes()),
  462. undefined,
  463. options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
  464. );
  465. walValve.start();
  466. }
  467. try {
  468. const before = this.queries.getNodeAndEdgeCount();
  469. // Mark the index as in-flight BEFORE any writes: a run killed
  470. // mid-index (OOM, SIGKILL, the #850 liveness watchdog) leaves this
  471. // marker behind, so `codegraph status` can tell a truncated index
  472. // from a completed one instead of silently serving partial results.
  473. try { this.queries.setMetadata('index_state', 'indexing'); } catch { /* metadata is advisory */ }
  474. // Segment vocabulary starts empty and is repopulated by the node write
  475. // path as every file (re-)indexes below — so a full index is also the
  476. // orphan-cleanup pass for names deleted since the last one.
  477. try { this.queries.clearNameSegmentVocab(); } catch { /* vocab is advisory — never fail an index over it */ }
  478. // Bulk FTS mode for the mass-insert phase: drop the per-row FTS sync
  479. // triggers, rebuild nodes_fts once from the nodes table afterwards.
  480. // Crash inside the window is healed on the next DatabaseConnection.open.
  481. this.db.beginBulkNodeLoad();
  482. // Fresh-init only: also drop the parse-lane secondary indexes for the
  483. // mass insert (the store-writer's B-tree-maintenance floor, plan §4d)
  484. // and rebuild each in one scan afterwards. Incremental runs keep them
  485. // — they delete per-file rows mid-phase through the file_path indexes.
  486. if (freshDb) this.db.beginBulkParseLoad();
  487. let result: IndexResult;
  488. try {
  489. result = await this.orchestrator.indexAll(
  490. options.onProgress,
  491. options.signal,
  492. options.verbose,
  493. walValve ? () => walValve!.backpressure() : undefined,
  494. // Store-writer offload is fresh-DB-only: with any pre-existing
  495. // data the store path must read (existing-file checks, cross-file
  496. // edge snapshots) and delete, which belongs on one thread.
  497. freshDb ? { dbPath: this.db.getPath(), fastInit } : null
  498. );
  499. } finally {
  500. if (freshDb) {
  501. const tIdx = Date.now();
  502. await this.db.endBulkParseLoad();
  503. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] parse-index-rebuild: ${Date.now() - tIdx}ms`);
  504. }
  505. const tFts = Date.now();
  506. this.db.endBulkNodeLoad();
  507. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] fts-rebuild: ${Date.now() - tFts}ms`);
  508. }
  509. // Fold the parse phase's WAL BEFORE the first post-parse reads
  510. // (resolver re-init and resolution both read on the main thread):
  511. // paging a bulk-write-sized WAL there is what blew the #850
  512. // watchdog's 60s window in the #1231 repro. Off-thread + awaited,
  513. // so the event loop keeps turning.
  514. if (walValve) await walValve.foldNow();
  515. // Re-detect frameworks now that the index is populated. The resolver
  516. // is constructed with createResolver() before any files exist, so
  517. // framework resolvers whose detect() consults the indexed file list
  518. // (e.g. UIKit/SwiftUI scanning for imports, swift-objc-bridge looking
  519. // for both Swift and ObjC files) all return false on that initial pass
  520. // and silently drop themselves. Re-initializing here gives them a
  521. // chance to see the actual project before resolution runs.
  522. if (result.success && result.filesIndexed > 0) {
  523. const tReinit = Date.now();
  524. this.resolver.initialize();
  525. // Cross-file finalization (e.g. NestJS RouterModule prefixes). Runs
  526. // before resolution so updated names show up in subsequent reads.
  527. this.resolver.runPostExtract();
  528. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] resolver-reinit: ${Date.now() - tReinit}ms`);
  529. }
  530. // Resolve references to create call/import/extends edges
  531. if (result.success && result.filesIndexed > 0) {
  532. // Get count without loading all refs into memory
  533. const unresolvedCount = this.queries.getUnresolvedReferencesCount();
  534. // Fast-init leaves the DB in memory-journal (rollback) mode, where
  535. // the parallel resolver pool's read connections would contend with
  536. // the main writer's exclusive commits. When the pool will actually
  537. // run (enough pending refs), restore WAL BEFORE resolution so
  538. // readers never block the writer; otherwise stay in the fast mode
  539. // until the finally — sequential resolution has no readers.
  540. if (fastInit && unresolvedCount >= minRefsForPool()) {
  541. try {
  542. this.db.getDb().pragma('synchronous = NORMAL');
  543. this.db.getDb().pragma('journal_mode = WAL');
  544. // Defer auto-checkpointing for the resolution phase, same
  545. // rationale as the deferWal path above: at the default 1000-page
  546. // interval, the persist loop's edge inserts + ref deletes make
  547. // SQLite re-write hot B-tree pages into the main DB file inline
  548. // on the writer over and over (#1231's pathology — measured as
  549. // ~58% of the resolution phase on a 255k-ref repo). The valve
  550. // bounds WAL growth off-thread; runMaintenance does the final
  551. // fold and the finally restores the interval.
  552. priorAutocheckpoint = this.db.getWalAutocheckpoint();
  553. this.db.setWalAutocheckpoint(0);
  554. restoreAutocheckpoint = true;
  555. walValve = new WalCheckpointValve(
  556. this.db,
  557. undefined,
  558. undefined,
  559. options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
  560. );
  561. walValve.start();
  562. } catch { /* keep current mode; resolution still works sequentially */ }
  563. }
  564. options.onProgress?.({
  565. phase: 'resolving',
  566. current: 0,
  567. total: unresolvedCount,
  568. });
  569. const tResolve = Date.now();
  570. await this.resolveReferencesBatched(
  571. (current, total) => {
  572. options.onProgress?.({
  573. phase: 'resolving',
  574. current,
  575. total,
  576. });
  577. },
  578. (done, totalPasses) => {
  579. options.onProgress?.({
  580. phase: 'linking',
  581. current: done,
  582. total: totalPasses,
  583. });
  584. },
  585. walValve ? () => walValve!.backpressure() : undefined
  586. );
  587. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] resolution: ${Date.now() - tResolve}ms`);
  588. // Second pass: chained calls whose method lives on a supertype the
  589. // receiver conforms to (protocol-extension / inherited / default-
  590. // interface). Needs the implements/extends edges the main pass just
  591. // built, so it runs after resolution (#750).
  592. const tChained = Date.now();
  593. await this.resolver.resolveChainedCallsViaConformance();
  594. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[synth-timing] chainedConformance: ${Date.now() - tChained}ms`);
  595. // Same lifecycle for `this.<member>` callback registrations whose
  596. // member is inherited from a supertype (#808).
  597. const tDeferred = Date.now();
  598. await this.resolver.resolveDeferredThisMemberRefs();
  599. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[synth-timing] deferredThisMember: ${Date.now() - tDeferred}ms`);
  600. }
  601. // Refresh planner stats + checkpoint the WAL after bulk writes.
  602. // Off-thread (worker connection): on a multi-GB index this is minutes
  603. // of IO, and inline it starved the #850 watchdog AFTER a fully
  604. // successful index. Never load-bearing for correctness.
  605. if (result.success && result.filesIndexed > 0) {
  606. const tMaint = Date.now();
  607. // Quiesce the valve first so its in-flight checkpoint and the
  608. // maintenance checkpoint don't contend for the checkpointer lock
  609. // (the loser would silently no-op and leave the WAL unfolded).
  610. if (walValve) { walValve.stop(); await walValve.drain(); }
  611. await this.db.runMaintenance();
  612. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] maintenance: ${Date.now() - tMaint}ms`);
  613. }
  614. // The orchestrator only sees extraction-phase counts; resolution and
  615. // synthesizer edges (often >50% of the graph on JVM repos) come later.
  616. // Recompute against the DB so the CLI summary reports the true totals.
  617. if (result.success && result.filesIndexed > 0) {
  618. const tCount = Date.now();
  619. const after = this.queries.getNodeAndEdgeCount();
  620. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] count-recompute: ${Date.now() - tCount}ms`);
  621. result.nodesCreated = after.nodes - before.nodes;
  622. result.edgesCreated = after.edges - before.edges;
  623. }
  624. // Stamp the index with the engine that built it, so `codegraph status`
  625. // and `codegraph upgrade` can recommend a re-index when the running
  626. // engine produces richer extraction than the one on disk. Only on a
  627. // real full index — a sync touches a subset, so it must NOT advance the
  628. // extraction stamp (the bulk would still be stale). See extraction-version.ts.
  629. if (result.success && result.filesIndexed > 0) {
  630. try {
  631. this.queries.setMetadata('indexed_with_version', CodeGraphPackageVersion);
  632. this.queries.setMetadata('indexed_with_extraction_version', String(EXTRACTION_VERSION));
  633. } catch { /* metadata is advisory — never fail an index over it */ }
  634. }
  635. // Reconcile the scan's ground truth against what the pipeline
  636. // accounted for. A shortfall means files were silently dropped
  637. // (observed in the wild: a run under heavy load came up 37 files
  638. // short with no error) — record it and tell the user, don't let the
  639. // index pass as complete.
  640. try {
  641. if (!result.success) {
  642. this.queries.setMetadata('index_state', 'failed');
  643. } else {
  644. const accounted = result.filesIndexed + result.filesSkipped + result.filesErrored;
  645. const discovered = result.filesDiscovered;
  646. const shortfall = discovered !== undefined ? discovered - accounted : 0;
  647. if (discovered !== undefined && shortfall > 0) {
  648. this.queries.setMetadata('index_state', 'partial');
  649. this.queries.setMetadata('index_files_discovered', String(discovered));
  650. this.queries.setMetadata('index_files_accounted', String(accounted));
  651. result.errors.push({
  652. message: `Index is missing ${shortfall} of ${discovered} discovered files (indexed ${result.filesIndexed}, skipped ${result.filesSkipped}, errored ${result.filesErrored}). The index is PARTIAL — re-run \`codegraph index\`.`,
  653. severity: 'warning',
  654. code: 'index_partial',
  655. });
  656. } else {
  657. this.queries.setMetadata('index_state', 'complete');
  658. if (discovered !== undefined) {
  659. this.queries.setMetadata('index_files_discovered', String(discovered));
  660. this.queries.setMetadata('index_files_accounted', String(accounted));
  661. }
  662. }
  663. }
  664. } catch { /* metadata is advisory — never fail an index over it */ }
  665. return result;
  666. } finally {
  667. // Restore the auto-checkpoint interval AFTER the fold-up above so the
  668. // next ordinary write doesn't inherit a giant inline checkpoint. On
  669. // the error path the WAL may still be large; correctness is unchanged
  670. // (SQLite replays the WAL on the next open) and the follow-up write
  671. // that folds it is the known cost of a failed run.
  672. if (walValve) { walValve.stop(); await walValve.drain(); }
  673. if (deferWal || restoreAutocheckpoint) {
  674. try { this.db.setWalAutocheckpoint(priorAutocheckpoint); } catch { /* connection may be closing */ }
  675. }
  676. if (fastInit) {
  677. // Back to the durable defaults; journal_mode=WAL folds the MEMORY
  678. // journal state into a normal WAL-mode database file.
  679. try {
  680. this.db.getDb().pragma('synchronous = NORMAL');
  681. this.db.getDb().pragma('journal_mode = WAL');
  682. } catch { /* connection may be closing */ }
  683. }
  684. this.fileLock.release();
  685. }
  686. });
  687. }
  688. /**
  689. * Index specific files
  690. *
  691. * Uses a mutex to prevent concurrent indexing operations.
  692. */
  693. async indexFiles(filePaths: string[]): Promise<IndexResult> {
  694. return this.indexMutex.withLock(async () => {
  695. try {
  696. this.fileLock.acquire();
  697. } catch {
  698. return { success: false, filesIndexed: 0, filesSkipped: 0, filesErrored: 0, nodesCreated: 0, edgesCreated: 0, errors: [{ message: 'Could not acquire file lock - another process may be indexing', severity: 'error' as const }], durationMs: 0 };
  699. }
  700. try {
  701. return this.orchestrator.indexFiles(filePaths);
  702. } finally {
  703. this.fileLock.release();
  704. }
  705. });
  706. }
  707. /**
  708. * Sync with current file state (incremental update)
  709. *
  710. * Uses a mutex to prevent concurrent indexing operations.
  711. */
  712. async sync(options: IndexOptions = {}): Promise<SyncResult> {
  713. return this.indexMutex.withLock(async () => {
  714. try {
  715. this.fileLock.acquire();
  716. } catch {
  717. return { filesChecked: 0, filesAdded: 0, filesModified: 0, filesRemoved: 0, nodesUpdated: 0, durationMs: 0 };
  718. }
  719. // Defer WAL auto-checkpointing for the whole incremental run, exactly
  720. // as indexAll does for the bulk path (#1231): sync's store loop and its
  721. // resolution passes churn the same FTS + secondary-index hot pages, and
  722. // at the default 1000-page cadence the inline checkpoints re-write them
  723. // over and over — on HDD-class storage a 7-file sync took 2 minutes at
  724. // 0-2% CPU (#1248). The cost scales with the EXISTING database size,
  725. // not the change size, so small syncs on big indexes hurt most. The
  726. // valve bounds WAL growth off-thread; runMaintenance at the end does
  727. // the final fold-up before the interval is restored in the finally.
  728. // Same kill switch as indexAll: CODEGRAPH_NO_WAL_DEFER=1. Idle valve
  729. // cost is one timer, so watcher-frequency syncs stay cheap.
  730. const deferWal = process.env.CODEGRAPH_NO_WAL_DEFER !== '1' && this.db.getJournalMode() === 'wal';
  731. let walValve: WalCheckpointValve | null = null;
  732. let priorAutocheckpoint = 1000;
  733. if (deferWal) {
  734. priorAutocheckpoint = this.db.getWalAutocheckpoint();
  735. this.db.setWalAutocheckpoint(0);
  736. walValve = new WalCheckpointValve(
  737. this.db,
  738. resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB, this.db.getDbFileSizeBytes()),
  739. undefined,
  740. options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
  741. );
  742. walValve.start();
  743. }
  744. try {
  745. // Captured BEFORE the sync runs: the sync's own incremental writes
  746. // populate vocab rows for the files it touches, so an end-of-sync
  747. // emptiness check would see "non-empty" and skip the backfill forever,
  748. // leaving every unchanged file's names unsegmented.
  749. const vocabWasEmpty = (() => {
  750. try { return this.queries.isNameSegmentVocabEmpty(); } catch { return false; }
  751. })();
  752. const result = await this.orchestrator.sync(options.onProgress, options.paths);
  753. // Fold the store phase's WAL BEFORE the post-store reads below
  754. // (resolution reads on the main thread) — same rationale as
  755. // indexAll's fold between store and resolution.
  756. if (walValve) await walValve.foldNow();
  757. // Cross-file finalization (e.g. NestJS RouterModule prefixes). Run on
  758. // every sync that touched files so edits to `app.module.ts` propagate
  759. // to controllers in unchanged files. The pass is idempotent and cheap
  760. // (regex over *.module.ts only).
  761. if (result.filesAdded > 0 || result.filesModified > 0) {
  762. this.resolver.runPostExtract();
  763. } else if (result.filesRemoved > 0) {
  764. // A pure-removal sync still resolves refs below — the deletion path
  765. // resurrects the removed file's incoming edges as pending refs
  766. // (#1240 removal case) and the orphan sweep consumes them. In a
  767. // long-lived process (daemon) the resolver's name caches were
  768. // warmed against the pre-removal graph; drop them so resolution
  769. // sees the post-removal state. (runPostExtract above clears caches
  770. // itself, so the changed-files branch is already covered.)
  771. this.resolver.clearCaches();
  772. }
  773. // Resolve references if files were updated
  774. const filesChanged = result.filesAdded > 0 || result.filesModified > 0;
  775. if (filesChanged) {
  776. if (result.changedFilePaths) {
  777. // Scope resolution to changed files (git fast path — bounded set)
  778. const tRefLoad = Date.now();
  779. const unresolvedRefs = this.queries.getUnresolvedReferencesByFiles(result.changedFilePaths);
  780. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-ref-load: ${Date.now() - tRefLoad}ms (${unresolvedRefs.length} refs)`);
  781. options.onProgress?.({
  782. phase: 'resolving',
  783. current: 0,
  784. total: unresolvedRefs.length,
  785. });
  786. this.resolver.resolveAndPersist(unresolvedRefs, (current, total) => {
  787. options.onProgress?.({
  788. phase: 'resolving',
  789. current,
  790. total,
  791. });
  792. });
  793. // Retry previously-failed refs the changed files may now satisfy
  794. // (#1240). Scoped resolution above only re-resolves refs FROM the
  795. // changed files — but when a changed file gains an export/symbol,
  796. // refs in UNCHANGED files that failed against the old graph can
  797. // now resolve, and nothing else ever revisits them (their rows
  798. // were parked as status='failed' by an earlier completed pass).
  799. // Look them up by the symbol names the changed files now carry
  800. // and re-resolve just that set. On a sync where no failed ref
  801. // matches, this is one indexed lookup.
  802. const tRetry = Date.now();
  803. const retryable = this.queries.getRetryableFailedReferences(
  804. this.queries.getNodeNamesByFiles(result.changedFilePaths)
  805. );
  806. if (retryable.length > 0) {
  807. options.onProgress?.({
  808. phase: 'resolving',
  809. current: 0,
  810. total: retryable.length,
  811. });
  812. await this.resolver.resolveAndPersistListYielding(retryable);
  813. options.onProgress?.({
  814. phase: 'resolving',
  815. current: retryable.length,
  816. total: retryable.length,
  817. });
  818. }
  819. if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-failed-ref-retry: ${Date.now() - tRetry}ms (${retryable.length} refs)`);
  820. } else {
  821. // No git info — use batched resolution to avoid OOM
  822. const unresolvedCount = this.queries.getUnresolvedReferencesCount();
  823. options.onProgress?.({
  824. phase: 'resolving',
  825. current: 0,
  826. total: unresolvedCount,
  827. });
  828. await this.resolveReferencesBatched(
  829. (current, total) => {
  830. options.onProgress?.({
  831. phase: 'resolving',
  832. current,
  833. total,
  834. });
  835. },
  836. (done, totalPasses) => {
  837. options.onProgress?.({
  838. phase: 'linking',
  839. current: done,
  840. total: totalPasses,
  841. });
  842. }
  843. );
  844. }
  845. }
  846. // Re-open resolution edges this sync may have invalidated ELSEWHERE in
  847. // the repo (CG-33). Everything above re-resolves references in the
  848. // changed files; this covers the opposite direction — references in
  849. // files the sync never touched whose answer depended on a definition
  850. // that just appeared or disappeared. Without it a synced index never
  851. // converges to a full rebuild: measured at 4.3% of distinct edges wrong
  852. // on codegraph's own index, in both directions, mostly `calls`. The
  853. // resurrected refs are pending rows, so the orphan sweep immediately
  854. // below is what resolves them — batched, yielding, multi-pass, exactly
  855. // as a full index resolves.
  856. //
  857. // `definitionDelta` is empty for a body-only edit, so the overwhelmingly
  858. // common sync pays one branch. CODEGRAPH_NO_REBIND=1 disables it.
  859. if (result.definitionDelta && process.env.CODEGRAPH_NO_REBIND !== '1') {
  860. const tRebind = Date.now();
  861. const rebound = this.orchestrator.resurrectStaleResolutionEdges(
  862. result.definitionDelta,
  863. result.changedFilePaths ?? []
  864. );
  865. if (process.env.CODEGRAPH_SYNTH_TIMINGS) {
  866. console.error(
  867. `[phase-timing] sync-rebind: ${Date.now() - tRebind}ms (${result.definitionDelta.length} changed names, ${rebound} edges re-opened)`
  868. );
  869. }
  870. }
  871. // Orphan sweep (#1187). A resolution pass that dies mid-run — the #850
  872. // daemon liveness watchdog's SIGKILL (#1122), Ctrl-C, a crash — leaves
  873. // the refs it never reached in unresolved_refs, and the git-scoped fast
  874. // path above never revisits them (it reads only the changed files'
  875. // rows). Those files' call edges were then missing PERMANENTLY, with
  876. // nothing to see except a too-small blast radius, until a full
  877. // re-index. A completed pass takes every row it processed out of the
  878. // PENDING set (resolved rows are deleted, unresolvable ones parked as
  879. // status='failed' for the #1240 retry above), so any pending row now
  880. // is such an orphan — or a row from an older engine's scoped pass.
  881. // Grind them down with the batched resolver; this also makes a bare
  882. // `codegraph sync` the recovery command for a wedged index. On a
  883. // healthy index this is one COUNT query.
  884. const orphanCount = this.queries.getUnresolvedReferencesCount();
  885. if (orphanCount > 0) {
  886. options.onProgress?.({
  887. phase: 'resolving',
  888. current: 0,
  889. total: orphanCount,
  890. });
  891. await this.resolveReferencesBatched(
  892. (current, total) => {
  893. options.onProgress?.({
  894. phase: 'resolving',
  895. current,
  896. total,
  897. });
  898. },
  899. (done, totalPasses) => {
  900. options.onProgress?.({
  901. phase: 'linking',
  902. current: done,
  903. total: totalPasses,
  904. });
  905. }
  906. );
  907. }
  908. if (filesChanged || orphanCount > 0) {
  909. // Second pass: chained calls whose method lives on a supertype the
  910. // receiver conforms to (protocol-extension / inherited). Needs the
  911. // implements/extends edges built above (#750).
  912. await this.resolver.resolveChainedCallsViaConformance();
  913. // Same lifecycle for `this.<member>` callback registrations whose
  914. // member is inherited from a supertype (#808).
  915. await this.resolver.resolveDeferredThisMemberRefs();
  916. }
  917. // Refresh planner stats + checkpoint the WAL after bulk writes.
  918. // Off-thread — see indexAll's call site.
  919. if (filesChanged || result.filesRemoved > 0 || orphanCount > 0) {
  920. await this.db.runMaintenance();
  921. }
  922. // Heal the segment vocabulary on indexes built before the table
  923. // existed (upgrade path): incremental writes above only cover changed
  924. // files, so a vocab that was empty when this sync STARTED means the
  925. // bulk was never segmented — backfill it (INSERT OR IGNORE, so the
  926. // rows the sync just wrote are fine). Batched + yielding — sync can
  927. // run on the daemon's liveness-watchdog thread (#850/#1091).
  928. try {
  929. if (vocabWasEmpty && this.queries.getNodeAndEdgeCount().nodes > 0) {
  930. await this.rebuildNameSegmentVocab();
  931. }
  932. } catch { /* vocab is advisory — never fail a sync over it */ }
  933. // A killed full index leaves this marker at `indexing`. Sync repairs
  934. // missing files, pending refs, and (on open) dropped indexes, so a
  935. // successful recovery must also close the metadata state (#1556).
  936. const fullReconcile = !options.paths || options.paths.length === 0;
  937. if (fullReconcile && this.getIndexState() === 'indexing') {
  938. try { this.queries.setMetadata('index_state', 'complete'); } catch { /* advisory */ }
  939. }
  940. return result;
  941. } finally {
  942. // Mirror indexAll's teardown: stop the valve, then restore the
  943. // auto-checkpoint interval (runMaintenance above already folded the
  944. // WAL on the success path; on the error path SQLite replays it on
  945. // the next open).
  946. if (walValve) { walValve.stop(); await walValve.drain(); }
  947. if (deferWal) {
  948. try { this.db.setWalAutocheckpoint(priorAutocheckpoint); } catch { /* connection may be closing */ }
  949. }
  950. this.fileLock.release();
  951. }
  952. });
  953. }
  954. /**
  955. * Check if an indexing operation is currently in progress
  956. */
  957. isIndexing(): boolean {
  958. return this.indexMutex.isLocked();
  959. }
  960. // ===========================================================================
  961. // File Watching
  962. // ===========================================================================
  963. /**
  964. * Start watching for file changes and auto-syncing.
  965. *
  966. * Uses native OS file events (FSEvents on macOS, inotify on Linux 19+,
  967. * ReadDirectoryChangesW on Windows) with debouncing to avoid thrashing.
  968. *
  969. * @param options - Watch options (debounce delay, callbacks)
  970. * @returns true if watching started successfully
  971. */
  972. watch(options: WatchOptions = {}): boolean {
  973. if (this.watcher?.isActive()) return true;
  974. this.watcher = new FileWatcher(
  975. this.projectRoot,
  976. async (paths?: string[]) => {
  977. const result = await this.sync({ paths });
  978. // sync() returns this exact zero-shape iff it failed to acquire the
  979. // file lock (a real empty sync always has filesChecked > 0 because
  980. // scanDirectory ran). Surface that to the watcher as a typed error
  981. // so it keeps pendingFiles + reschedules instead of clearing them
  982. // (#449).
  983. if (result.filesChecked === 0 && result.durationMs === 0) {
  984. throw new LockUnavailableError();
  985. }
  986. const filesChanged = result.filesAdded + result.filesModified + result.filesRemoved;
  987. return { filesChanged, durationMs: result.durationMs };
  988. },
  989. options
  990. );
  991. return this.watcher.start();
  992. }
  993. /**
  994. * Stop watching for file changes.
  995. */
  996. unwatch(): void {
  997. if (this.watcher) {
  998. this.watcher.stop();
  999. this.watcher = null;
  1000. }
  1001. }
  1002. /**
  1003. * Check if the file watcher is active.
  1004. */
  1005. isWatching(): boolean {
  1006. return this.watcher?.isActive() ?? false;
  1007. }
  1008. /**
  1009. * True once live watching has permanently degraded (OS watch-resource
  1010. * exhaustion, or a write lock held past the retry budget) and auto-sync is
  1011. * disabled until the next {@link watch} call. Distinct from `!isWatching()`:
  1012. * a stopped/never-started watcher is inactive but NOT degraded. MCP tools use
  1013. * this to surface a whole-index "results may be stale" notice, since
  1014. * `getPendingFiles()` goes empty once watching stops (#876).
  1015. */
  1016. isWatcherDegraded(): boolean {
  1017. return this.watcher?.isDegraded() ?? false;
  1018. }
  1019. /** The reason live watching degraded, or null if it is healthy (#876). */
  1020. getWatcherDegradedReason(): string | null {
  1021. return this.watcher?.getDegradedReason() ?? null;
  1022. }
  1023. /**
  1024. * Files seen by the file watcher since the last successful sync —
  1025. * the per-file "stale" signal MCP tools attach to responses so an agent
  1026. * can fall back to {@link Read} for just the affected file without
  1027. * waiting for a debounced sync to complete (issue #403).
  1028. *
  1029. * Returns an empty list when the watcher isn't active, or no events have
  1030. * arrived. Each entry includes `firstSeenMs` and `lastSeenMs` (wall-clock
  1031. * `Date.now()` values) so callers can render "edited Nms ago", plus an
  1032. * `indexing` flag indicating whether the in-flight sync (if any) will
  1033. * absorb that file.
  1034. */
  1035. getPendingFiles(): PendingFile[] {
  1036. return this.watcher?.getPendingFiles() ?? [];
  1037. }
  1038. /**
  1039. * Resolves once the file watcher has installed its watch set. Useful for
  1040. * tests that need a deterministic boundary before asserting on
  1041. * `getPendingFiles()`. Resolves immediately when no watcher is active.
  1042. */
  1043. waitUntilWatcherReady(timeoutMs?: number): Promise<void> {
  1044. return this.watcher ? this.watcher.waitUntilReady(timeoutMs) : Promise.resolve();
  1045. }
  1046. /**
  1047. * Get files that have changed since last index
  1048. */
  1049. getChangedFiles(): { added: string[]; modified: string[]; removed: string[] } {
  1050. return this.orchestrator.getChangedFiles();
  1051. }
  1052. /**
  1053. * Most recent index timestamp (ms since epoch) across all tracked files, or
  1054. * null when nothing is indexed yet. Lets library consumers check index
  1055. * freshness without shelling out to `codegraph status --json`. (#329)
  1056. */
  1057. getLastIndexedAt(): number | null {
  1058. return this.queries.getLastIndexedAt();
  1059. }
  1060. /**
  1061. * How far the last sync got and how many files it left behind — the cheapest
  1062. * marker of "has this index moved". One query; safe to call on every
  1063. * filesystem event a live viewer sees.
  1064. */
  1065. getIndexRevision(): { lastIndexedAt: number | null; fileCount: number } {
  1066. return this.queries.getIndexRevision();
  1067. }
  1068. /**
  1069. * Files re-indexed strictly after `since`, newest first — what a sync just
  1070. * picked up. `total` is the real count, `paths` is capped at `limit`.
  1071. */
  1072. getFilesIndexedSince(since: number, limit: number): { paths: string[]; total: number } {
  1073. return this.queries.getFilesIndexedSince(since, limit);
  1074. }
  1075. /**
  1076. * Forget everything held in memory about rows another process may have
  1077. * changed.
  1078. *
  1079. * The query layer keeps an LRU of nodes by id, invalidated by writes made
  1080. * through THIS instance — which is exactly right for a process that owns the
  1081. * index, and wrong for one that is only reading a database somebody else is
  1082. * writing. A long-lived reader (the `codegraph ui` server, a daemon holding a
  1083. * graph open across an agent's edits) will otherwise answer `getNode(id)`
  1084. * with a row a sync deleted minutes ago, while every SQL-backed query beside
  1085. * it reports the truth — a disagreement that reads as a bug in whichever
  1086. * screen shows both.
  1087. *
  1088. * Cheap (clearing a bounded Map) and safe to call whenever the database file
  1089. * looks like it moved.
  1090. */
  1091. dropReadCaches(): void {
  1092. this.queries.clearCache();
  1093. }
  1094. /**
  1095. * Completeness of the last full index run. `'complete'` is the only good
  1096. * state. `'indexing'` after the fact means a run was killed mid-index (OOM,
  1097. * SIGKILL, liveness watchdog) and the on-disk index is truncated;
  1098. * `'partial'` means the run finished but silently dropped files
  1099. * (discovered > indexed+skipped+errored); `'failed'` means it reported
  1100. * failure. `null` = index predates this marker. Surfaced by
  1101. * `codegraph status`.
  1102. */
  1103. getIndexState(): 'indexing' | 'complete' | 'partial' | 'failed' | null {
  1104. const raw = this.queries.getMetadata('index_state');
  1105. return raw === 'indexing' || raw === 'complete' || raw === 'partial' || raw === 'failed'
  1106. ? raw
  1107. : null;
  1108. }
  1109. /**
  1110. * Which engine built the current index: the package version + extraction
  1111. * version stamped at the last full `indexAll`. Either field is null for an
  1112. * index built before stamping existed (treated as stale). See
  1113. * `extraction-version.ts` and `isIndexStale()`.
  1114. */
  1115. getIndexBuildInfo(): { version: string | null; extractionVersion: number | null } {
  1116. const version = this.queries.getMetadata('indexed_with_version');
  1117. const ev = this.queries.getMetadata('indexed_with_extraction_version');
  1118. const parsed = ev != null ? parseInt(ev, 10) : NaN;
  1119. return { version, extractionVersion: Number.isFinite(parsed) ? parsed : null };
  1120. }
  1121. /**
  1122. * True when the on-disk index was built by an engine whose extraction is
  1123. * older than the one now running — i.e. a re-index would add data a migration
  1124. * can't backfill. False when there's no index yet (nothing to refresh) or the
  1125. * stamp is current. This is the signal behind `codegraph status`'s re-index
  1126. * hint and `codegraph upgrade`'s reminder.
  1127. */
  1128. isIndexStale(): boolean {
  1129. if (this.queries.getLastIndexedAt() == null) return false;
  1130. const { extractionVersion } = this.getIndexBuildInfo();
  1131. return extractionVersion == null || extractionVersion < EXTRACTION_VERSION;
  1132. }
  1133. /**
  1134. * Extract nodes and edges from source code (without storing)
  1135. */
  1136. extractFromSource(filePath: string, source: string): ExtractionResult {
  1137. return extractFromSource(filePath, source);
  1138. }
  1139. // ===========================================================================
  1140. // Reference Resolution
  1141. // ===========================================================================
  1142. /**
  1143. * Resolve unresolved references and create edges
  1144. *
  1145. * This method takes unresolved references from extraction and attempts
  1146. * to resolve them using multiple strategies:
  1147. * - Framework-specific patterns (React, Express, Laravel)
  1148. * - Import-based resolution
  1149. * - Name-based symbol matching
  1150. */
  1151. resolveReferences(onProgress?: (current: number, total: number) => void): ResolutionResult {
  1152. // Get all unresolved references from the database
  1153. const unresolvedRefs = this.queries.getUnresolvedReferences();
  1154. return this.resolver.resolveAndPersist(unresolvedRefs, onProgress);
  1155. }
  1156. /**
  1157. * Resolve references in batches to keep memory bounded on large codebases.
  1158. * Processes chunks of unresolved refs, persisting results after each batch.
  1159. */
  1160. async resolveReferencesBatched(
  1161. onProgress?: (current: number, total: number) => void,
  1162. onSynthesisProgress?: (done: number, total: number) => void,
  1163. // The WAL valve's writer-side backstop, threaded into the batch loop's
  1164. // pool-idle boundaries. Without it the valve's only lever during
  1165. // resolution is timer-driven passive checkpoints, which the pool's
  1166. // continuous reads keep perpetually partial — the WAL then accretes the
  1167. // whole phase's write volume (22GB on a 4.6GB DB at kernel scale).
  1168. backpressure?: () => Promise<void> | null
  1169. ): Promise<ResolutionResult> {
  1170. return this.resolver.resolveAndPersistBatched(onProgress, undefined, onSynthesisProgress, {
  1171. dbPath: this.db.getPath(),
  1172. // Bulk-edge-load hooks: on big runs the resolver drops the non-unique
  1173. // edge indexes for the batch loop and recreates them before synthesis
  1174. // (which reads kind-keyed). Concurrent readers (a daemon serving this
  1175. // project mid-index) stay CORRECT during the window — target/kind reads
  1176. // just degrade to scans until the recreate.
  1177. bulkEdgeLoad: {
  1178. begin: () => this.db.beginBulkEdgeLoad(),
  1179. end: () => this.db.endBulkEdgeLoad(),
  1180. },
  1181. refIndexLoad: {
  1182. begin: () => this.db.beginBulkRefLoad(),
  1183. end: () => this.db.endBulkRefLoad(),
  1184. },
  1185. backpressure,
  1186. });
  1187. }
  1188. /**
  1189. * References extracted but never attempted by a resolution pass. Zero on a
  1190. * healthy index — a completed pass consumes every pending row (resolving it
  1191. * or parking it as failed, #1240). Non-zero at rest means a pass was
  1192. * interrupted mid-run (killed indexer, crash — #1187), so some files' call
  1193. * edges are missing; the next `sync` sweeps them.
  1194. */
  1195. getPendingReferenceCount(): number {
  1196. return this.queries.getUnresolvedReferencesCount();
  1197. }
  1198. /**
  1199. * Get detected frameworks in the project
  1200. */
  1201. getDetectedFrameworks(): string[] {
  1202. return this.resolver.getDetectedFrameworks();
  1203. }
  1204. /**
  1205. * Re-initialize the resolver (useful after adding new files)
  1206. */
  1207. reinitializeResolver(): void {
  1208. this.resolver.initialize();
  1209. }
  1210. // ===========================================================================
  1211. // Graph Statistics
  1212. // ===========================================================================
  1213. /**
  1214. * Get statistics about the knowledge graph
  1215. */
  1216. getStats(): GraphStats {
  1217. const stats = this.queries.getStats();
  1218. stats.dbSizeBytes = this.db.getSize();
  1219. stats.walSizeBytes = this.db.getWalSizeBytes();
  1220. return stats;
  1221. }
  1222. /**
  1223. * Active SQLite backend for this project's connection (`node-sqlite` — Node's
  1224. * built-in real-SQLite module). Surfaced via `codegraph status` and the
  1225. * `codegraph_status` MCP tool alongside the effective journal mode.
  1226. */
  1227. getBackend(): import('./db').SqliteBackend {
  1228. return this.db.getBackend();
  1229. }
  1230. /**
  1231. * The journal mode actually in effect ('wal', 'delete', …). 'wal' means
  1232. * readers never block on a concurrent writer; anything else means they can,
  1233. * which is the precondition for the "database is locked" failures in issue
  1234. * #238. Surfaced via `codegraph status` and the `codegraph_status` MCP tool.
  1235. */
  1236. getJournalMode(): string {
  1237. return this.db.getJournalMode();
  1238. }
  1239. // ===========================================================================
  1240. // Node Operations
  1241. // ===========================================================================
  1242. /**
  1243. * Get a node by ID
  1244. */
  1245. getNode(id: string): Node | null {
  1246. return this.queries.getNodeById(id);
  1247. }
  1248. /**
  1249. * Get many nodes by id in ONE round-trip (LRU-cache aware).
  1250. *
  1251. * The batch form of {@link getNode}. Anything resolving a list of edges to
  1252. * their endpoints — a caller list, a callee rail, an impact set — must use
  1253. * this rather than a `getNode` per edge: a symbol with 500 callers is 500
  1254. * queries otherwise. Ids that name nothing are simply absent from the map.
  1255. */
  1256. getNodesByIds(ids: readonly string[]): Map<string, Node> {
  1257. return this.queries.getNodesByIds(ids);
  1258. }
  1259. /**
  1260. * Outgoing edges for many source nodes at once — the batch form of
  1261. * {@link getOutgoingEdges}. See {@link QueryBuilder.getOutgoingEdgesFrom}.
  1262. */
  1263. getOutgoingEdgesFrom(nodeIds: readonly string[], kinds?: Edge['kind'][]): Edge[] {
  1264. return this.queries.getOutgoingEdgesFrom(nodeIds, kinds);
  1265. }
  1266. /**
  1267. * Fan-in (incoming edge count) for many nodes at once — the "hub" signal,
  1268. * without a query per node. See {@link QueryBuilder.countIncomingEdges}.
  1269. */
  1270. getFanIn(ids: readonly string[]): Map<string, number> {
  1271. return this.queries.countIncomingEdges(ids);
  1272. }
  1273. /**
  1274. * Incoming edges for many target nodes at once — the mirror of
  1275. * {@link getOutgoingEdgesFrom}. See {@link QueryBuilder.getIncomingEdgesTo}.
  1276. */
  1277. getIncomingEdgesTo(nodeIds: readonly string[], kinds?: Edge['kind'][]): Edge[] {
  1278. return this.queries.getIncomingEdgesTo(nodeIds, kinds);
  1279. }
  1280. /**
  1281. * Fan-out (outgoing edge count) for many nodes at once — the mirror of
  1282. * {@link getFanIn}. See {@link QueryBuilder.countOutgoingEdges}.
  1283. */
  1284. getFanOut(ids: readonly string[]): Map<string, number> {
  1285. return this.queries.countOutgoingEdges(ids);
  1286. }
  1287. /**
  1288. * The symbols with the most distinct dependents, most first — the index's
  1289. * hubs. Distinct dependents, not edges: a helper called forty times from one
  1290. * function has one dependent, and it is dependents a blast radius grows from.
  1291. */
  1292. getTopDependedOn(limit: number): Array<{ nodeId: string; dependents: number }> {
  1293. return this.queries.getTopDependedOn(limit);
  1294. }
  1295. /**
  1296. * The graph's executable roots — files that run something at module level (a
  1297. * CLI, a worker entry, a script), ranked by calls x the number of other files
  1298. * they reach. A statement at the top level of a file is recorded as an edge
  1299. * out of the *file* node, which is what makes these visible at all.
  1300. */
  1301. getTopCallingFiles(
  1302. limit: number
  1303. ): Array<{ nodeId: string; filePath: string; calls: number; reaches: number; score: number }> {
  1304. return this.queries.getTopCallingFiles(limit);
  1305. }
  1306. /**
  1307. * How many other files depend on each of the given files, counted through
  1308. * their symbols (an `imports` edge points at the symbol, not the file).
  1309. * A zero means nothing else in the index reaches into that file.
  1310. */
  1311. getFileDependentCounts(filePaths: string[]): Map<string, number> {
  1312. return new Map(
  1313. this.queries.getFileDependentCounts(filePaths).map((row) => [row.filePath, row.dependents])
  1314. );
  1315. }
  1316. /**
  1317. * Roll the edge table up to module granularity, for a file → module
  1318. * assignment the caller decides.
  1319. *
  1320. * The architecture map's single query: cross-module edge counts by kind,
  1321. * the `declared` subset of each (see {@link QueryBuilder.aggregateModuleGraph}),
  1322. * and the busiest symbol pairs behind each link. Read-only, and bounded by
  1323. * the number of modules rather than the number of edges.
  1324. */
  1325. getModuleAggregation(
  1326. assignments: ReadonlyArray<{ filePath: string; module: string }>,
  1327. options: {
  1328. kinds: readonly Edge['kind'][];
  1329. minConfidence: number;
  1330. topPairsPerLink: number;
  1331. pairKinds: readonly Edge['kind'][];
  1332. }
  1333. ): ReturnType<QueryBuilder['aggregateModuleGraph']> {
  1334. return this.queries.aggregateModuleGraph(assignments, options);
  1335. }
  1336. /**
  1337. * Every ordered pair of files where one reaches into the other — the edge
  1338. * list a cycle finder runs on. See {@link QueryBuilder.getCrossFileDependencyPairs}.
  1339. */
  1340. getFileDependencyPairs(minConfidence = 0): Array<{ source: string; target: string }> {
  1341. return this.queries.getCrossFileDependencyPairs(minConfidence);
  1342. }
  1343. /**
  1344. * References from a symbol that never resolved to an indexed node — the
  1345. * calls and type mentions that leave the index. Lets a reader account for
  1346. * the call sites that have no callee row instead of implying there are none.
  1347. */
  1348. getUnresolvedReferencesFrom(nodeId: string): UnresolvedReference[] {
  1349. return this.queries.getUnresolvedReferencesFrom(nodeId);
  1350. }
  1351. /**
  1352. * The same, for every symbol in a FILE at once, in line order.
  1353. *
  1354. * One indexed lookup instead of one per symbol — the whole-file reader needs
  1355. * it for every line it draws. See
  1356. * {@link QueryBuilder.getUnresolvedReferencesInFile}.
  1357. */
  1358. getUnresolvedReferencesInFile(filePath: string, limit?: number): UnresolvedReference[] {
  1359. return this.queries.getUnresolvedReferencesInFile(filePath, limit);
  1360. }
  1361. /**
  1362. * Get all nodes in a file
  1363. */
  1364. getNodesInFile(filePath: string): Node[] {
  1365. return this.queries.getNodesByFile(filePath);
  1366. }
  1367. /**
  1368. * Get all nodes of a specific kind
  1369. */
  1370. getNodesByKind(kind: Node['kind']): Node[] {
  1371. return this.queries.getNodesByKind(kind);
  1372. }
  1373. /**
  1374. * Get ALL nodes with an exact name (direct index lookup, not FTS-ranked/capped).
  1375. * Used to enumerate every overload of a heavily-overloaded name so the specific
  1376. * definition the caller wants is never dropped below a search cut.
  1377. */
  1378. getNodesByName(name: string): Node[] {
  1379. return this.queries.getNodesByName(name);
  1380. }
  1381. /** Nodes whose name starts with `prefix` (index range scan, capped). */
  1382. getNodesByNamePrefix(prefix: string, limit = 20): Node[] {
  1383. return this.queries.getNodesByNamePrefix(prefix, limit);
  1384. }
  1385. /**
  1386. * Nodes whose name CONTAINS `substring` (LIKE scan, ASCII-case-insensitive,
  1387. * shortest-first). The camel-infix lookup FTS can't do — `profileInfo`
  1388. * inside `getProfileInfoV2` is one FTS token (#1196).
  1389. */
  1390. getNodesByNameSubstring(
  1391. substring: string,
  1392. options: { kinds?: NodeKind[]; limit?: number; excludePrefix?: boolean } = {}
  1393. ): Node[] {
  1394. return this.queries
  1395. .findNodesByNameSubstring(substring, options)
  1396. .map((r) => r.node);
  1397. }
  1398. /**
  1399. * Search nodes by text
  1400. */
  1401. searchNodes(query: string, options?: SearchOptions): SearchResult[] {
  1402. return this.queries.searchNodes(query, options);
  1403. }
  1404. /**
  1405. * Graph-derived prompt matching for the front-load hook's MEDIUM tier:
  1406. * which indexed symbols do these prose words name? "state machine des
  1407. * commandes" → `OrderStateMachine`, in any human language whose technical
  1408. * nouns are Latin script — no keyword list involved.
  1409. *
  1410. * Precision comes from the repo's own naming statistics, not vocabulary:
  1411. * - CO-OCCURRENCE: ≥2 words that are segments of the SAME name ("state" +
  1412. * "machine" → OrderStateMachine) is strong evidence and always qualifies.
  1413. * - RARITY: a single matched word qualifies only when its segment is
  1414. * discriminative here (≤ {@link SEGMENT_RARITY_CEILING} distinct names) —
  1415. * "checkout" in a shop backend yes, "state" in a react app no.
  1416. * Every candidate is re-verified against `nodes` before being returned
  1417. * (vocab rows are proposals; deletions leave orphans by design), so a
  1418. * returned symbol is guaranteed to exist right now.
  1419. */
  1420. getSegmentMatches(words: string[], limit: number = 6): SegmentMatch[] {
  1421. if (words.length === 0) return [];
  1422. // Variant → original word (plural folding), for coverage accounting.
  1423. const variantToWord = new Map<string, string>();
  1424. for (const word of words) {
  1425. for (const variant of segmentLookupVariants(word)) {
  1426. if (!variantToWord.has(variant)) variantToWord.set(variant, word);
  1427. }
  1428. }
  1429. const variants = [...variantToWord.keys()];
  1430. // Tier A: co-occurrence. The SQL folds variants back to their original
  1431. // word (#1146), so minWords=2 means two distinct PROMPT WORDS — a name
  1432. // matching both `service` and `services` can't tie with (or crowd past
  1433. // the LIMIT) a genuine two-word match. The JS re-check below recomputes
  1434. // the fold from live segments as the honesty layer.
  1435. const variantPairs = [...variantToWord.entries()].map(([segment, word]) => ({ segment, word }));
  1436. const candidates: Array<{ name: string; matchedWords: Set<string> }> = [];
  1437. for (const hit of this.queries.getSegmentCoOccurrence(variantPairs, 2, 24)) {
  1438. const matched = this.wordsMatchingName(hit.name, variantToWord);
  1439. if (matched.size >= 2) candidates.push({ name: hit.name, matchedWords: matched });
  1440. }
  1441. // Tier B: single rare word. Only when co-occurrence found nothing — a
  1442. // co-occurring name is categorically stronger evidence — and under
  1443. // stricter rules, because one word is thin: the word must be ≥5 chars
  1444. // (measured FPs: "this", "typo"); the segment must appear in AT LEAST TWO
  1445. // names (a concept the codebase is about clusters across names —
  1446. // CheckoutService/CheckoutController — while a prose coincidence is a
  1447. // singleton: measured FP "deploy to PRODUCTION" → the one name
  1448. // matchesNonProductionDir); and the candidate name must have ≥2 segments
  1449. // (a bare common verb matching a bare function name — "write" → `write` —
  1450. // is prose coincidence, not the user naming a symbol).
  1451. if (candidates.length === 0) {
  1452. const singleWordVariants = variants.filter((v) => variantToWord.get(v)!.length >= 5);
  1453. const counts = this.queries.getSegmentNameCounts(singleWordVariants);
  1454. const rare = [...counts.entries()]
  1455. .filter(([, n]) => n >= 2 && n <= CodeGraph.SEGMENT_RARITY_CEILING)
  1456. .sort((a, b) => a[1] - b[1])
  1457. .slice(0, 2);
  1458. for (const [variant] of rare) {
  1459. const word = variantToWord.get(variant)!;
  1460. for (const name of this.queries.getNamesForSegment(variant, 12)) {
  1461. if (splitIdentifierSegments(name).length < 2) continue;
  1462. candidates.push({ name, matchedWords: new Set([word]) });
  1463. }
  1464. }
  1465. }
  1466. // Verify against nodes (the honesty gate) and pick a representative
  1467. // definition per name. A name whose only nodes are file/import kind has
  1468. // no real definition to point at — surfacing the import statement instead
  1469. // reads as a matched symbol but isn't one (#1144) — so it's skipped, the
  1470. // same way an orphaned vocab row is. (Import names no longer enter the
  1471. // vocab at write time, but rows written before that exclusion persist
  1472. // until the next full index.)
  1473. const out: SegmentMatch[] = [];
  1474. const seen = new Set<string>();
  1475. candidates.sort((a, b) => b.matchedWords.size - a.matchedWords.size || a.name.length - b.name.length);
  1476. for (const candidate of candidates) {
  1477. if (out.length >= limit) break;
  1478. if (seen.has(candidate.name)) continue;
  1479. seen.add(candidate.name);
  1480. const nodes = this.queries.getNodesByName(candidate.name);
  1481. if (nodes.length === 0) continue; // orphaned vocab row — name no longer exists
  1482. const rep = nodes.find((n) => n.kind !== 'file' && n.kind !== 'import');
  1483. if (!rep) continue; // no real definition — don't surface an import/file as one
  1484. out.push({
  1485. name: candidate.name,
  1486. kind: rep.kind,
  1487. filePath: rep.filePath,
  1488. startLine: rep.startLine ?? 0,
  1489. matchedWords: [...candidate.matchedWords].sort(),
  1490. });
  1491. }
  1492. return out;
  1493. }
  1494. /** A single word ("state") can match hundreds of names in a big repo — that
  1495. * is noise, not signal. Ceiling for the single-word tier; co-occurrence is
  1496. * exempt because two words on one name is already discriminative. */
  1497. private static readonly SEGMENT_RARITY_CEILING = 25;
  1498. /** Which of the prompt's original words match `name`'s segments (via
  1499. * variants). Segments are recomputed in JS — a name-keyed vocab lookup
  1500. * would scan the (segment, name) primary key. */
  1501. private wordsMatchingName(name: string, variantToWord: Map<string, string>): Set<string> {
  1502. const segments = new Set(splitIdentifierSegments(name));
  1503. const matched = new Set<string>();
  1504. for (const [variant, word] of variantToWord) {
  1505. if (segments.has(variant)) matched.add(word);
  1506. }
  1507. return matched;
  1508. }
  1509. /**
  1510. * One-shot upgrade heal for callers that open the graph WITHOUT syncing —
  1511. * concretely the prompt hook, whose MEDIUM tier reads the segment
  1512. * vocabulary: a database migrated from before the vocab table existed
  1513. * starts with it empty, and the only other backfill lives inside `sync()`,
  1514. * which such callers never run (#1142). Returns true when the vocab is
  1515. * usable (already populated — the overwhelmingly common one-SELECT case —
  1516. * or healed here); false when it isn't (empty graph, or another process
  1517. * holds the index lock — that process's own sync heals it).
  1518. */
  1519. async healSegmentVocabIfEmpty(): Promise<boolean> {
  1520. const empty = (() => {
  1521. try { return this.queries.isNameSegmentVocabEmpty(); } catch { return false; }
  1522. })();
  1523. if (!empty) return true;
  1524. if (this.queries.getNodeAndEdgeCount().nodes === 0) return false;
  1525. return this.indexMutex.withLock(async () => {
  1526. try {
  1527. this.fileLock.acquire();
  1528. } catch {
  1529. return false; // an index/sync is running — it backfills the vocab itself
  1530. }
  1531. try {
  1532. if (!this.queries.isNameSegmentVocabEmpty()) return true; // raced: healed meanwhile
  1533. await this.rebuildNameSegmentVocab();
  1534. return true;
  1535. } finally {
  1536. this.fileLock.release();
  1537. }
  1538. });
  1539. }
  1540. /**
  1541. * Rebuild the segment vocabulary from the current graph, batched and
  1542. * yielding — the upgrade-heal path for indexes built before the vocab table
  1543. * existed. Runs inside the index mutex/lock (sync and
  1544. * healSegmentVocabIfEmpty hold them).
  1545. */
  1546. private async rebuildNameSegmentVocab(): Promise<void> {
  1547. const maybeYield = createYielder();
  1548. const BATCH = 2000;
  1549. for (let offset = 0; ; offset += BATCH) {
  1550. const names = this.queries.getDistinctNodeNames(BATCH, offset);
  1551. if (names.length === 0) break;
  1552. this.queries.insertNameSegmentsBatch(names);
  1553. await maybeYield();
  1554. }
  1555. }
  1556. /**
  1557. * Normalized project-name tokens (go.mod / package.json / repo dir) used to
  1558. * down-weight the non-discriminative project name in search ranking (#720).
  1559. * Exposed so explore can exclude it from the PascalCase type-disambiguation
  1560. * bias, which would otherwise pull overloaded tokens toward whichever stack
  1561. * embeds the project name.
  1562. */
  1563. getProjectNameTokens(): Set<string> {
  1564. return this.queries.getProjectNameTokens();
  1565. }
  1566. /**
  1567. * Find the project's "primary route file" — the file with the densest
  1568. * concentration of framework-emitted `route` nodes (≥3 routes, ≥30%
  1569. * of all non-test routes). Used to inline the routing config in
  1570. * `codegraph_explore` responses on small realworld template repos
  1571. * (rails-realworld, laravel-realworld, drupal-admintoolbar, …) where
  1572. * Glob+Read of `routes.rb`/`urls.py`/etc. otherwise beats codegraph.
  1573. */
  1574. getTopRouteFile(): { filePath: string; routeCount: number; totalRoutes: number } | null {
  1575. return this.queries.getTopRouteFile();
  1576. }
  1577. /**
  1578. * Build a URL → handler routing manifest from the index. Each entry
  1579. * pairs a route node (URL + method) with its handler function/method
  1580. * via the `references` edge that framework resolvers emit. Returns
  1581. * null when fewer than 3 valid (non-test) routes exist.
  1582. */
  1583. getRoutingManifest(limit?: number): {
  1584. entries: Array<{ url: string; handler: string; handlerFile: string; handlerLine: number; handlerKind: string }>;
  1585. topHandlerFile: string | null;
  1586. topHandlerFileCount: number;
  1587. totalRoutes: number;
  1588. } | null {
  1589. return this.queries.getRoutingManifest(limit);
  1590. }
  1591. // ===========================================================================
  1592. // Edge Operations
  1593. // ===========================================================================
  1594. /**
  1595. * Get outgoing edges from a node
  1596. */
  1597. getOutgoingEdges(nodeId: string): Edge[] {
  1598. return this.queries.getOutgoingEdges(nodeId);
  1599. }
  1600. /**
  1601. * Get incoming edges to a node
  1602. */
  1603. getIncomingEdges(nodeId: string): Edge[] {
  1604. return this.queries.getIncomingEdges(nodeId);
  1605. }
  1606. // ===========================================================================
  1607. // File Operations
  1608. // ===========================================================================
  1609. /**
  1610. * Get a file record by path
  1611. */
  1612. getFile(filePath: string): FileRecord | null {
  1613. return this.queries.getFileByPath(filePath);
  1614. }
  1615. /**
  1616. * Get all tracked files
  1617. */
  1618. getFiles(): FileRecord[] {
  1619. return this.queries.getAllFiles();
  1620. }
  1621. /**
  1622. * A `(path) => boolean` generated-file test over a BOUNDED candidate list,
  1623. * unioning the index-time content-banner flag with the filename convention
  1624. * (#1500). One query up front, O(1) per call after — built for use inside a
  1625. * ranking comparator, where re-querying per comparison would be quadratic.
  1626. *
  1627. * Pass every path you might ask about; a path outside the list falls back to
  1628. * the filename check alone.
  1629. */
  1630. generatedFilePredicate(filePaths: Iterable<string>): (filePath: string) => boolean {
  1631. return this.queries.generatedPredicateFor(filePaths);
  1632. }
  1633. /**
  1634. * A `(path) => boolean` ambient-declaration test over a BOUNDED candidate
  1635. * list: true for a file that declares nothing but types, originates no call
  1636. * edge, and that nothing in the index depends on — an ambient `.d.ts` of
  1637. * global shims, vendored typings, module augmentation (CG-28). Structural
  1638. * rather than extension-based, and deliberately narrow: see
  1639. * `QueryBuilder.getAmbientDeclarationPathsAmong` for why each condition is
  1640. * there, in particular why a `types.ts` the codebase imports is NOT flagged.
  1641. */
  1642. ambientDeclarationFilePredicate(filePaths: Iterable<string>): (filePath: string) => boolean {
  1643. return this.queries.ambientDeclarationPredicateFor(filePaths);
  1644. }
  1645. /** How many indexed files are flagged tool-generated. Reported by `status`. */
  1646. getGeneratedFileCount(): number {
  1647. return this.queries.countGeneratedFiles();
  1648. }
  1649. // ===========================================================================
  1650. // Graph Query Methods
  1651. // ===========================================================================
  1652. /**
  1653. * Get the context for a node (ancestors, children, references)
  1654. *
  1655. * Returns comprehensive context about a node including its containment
  1656. * hierarchy, children, incoming/outgoing references, type information,
  1657. * and relevant imports.
  1658. *
  1659. * @param nodeId - ID of the focal node
  1660. * @returns Context object with all related information
  1661. */
  1662. getContext(nodeId: string): Context {
  1663. return this.graphManager.getContext(nodeId);
  1664. }
  1665. /**
  1666. * Traverse the graph from a starting node
  1667. *
  1668. * Uses breadth-first search by default. Supports filtering by edge types,
  1669. * node types, and traversal direction.
  1670. *
  1671. * @param startId - Starting node ID
  1672. * @param options - Traversal options
  1673. * @returns Subgraph containing traversed nodes and edges
  1674. */
  1675. traverse(startId: string, options?: TraversalOptions): Subgraph {
  1676. return this.traverser.traverseBFS(startId, options);
  1677. }
  1678. /**
  1679. * Get the call graph for a function
  1680. *
  1681. * Returns both callers (functions that call this function) and
  1682. * callees (functions called by this function) up to the specified depth.
  1683. *
  1684. * @param nodeId - ID of the function/method node
  1685. * @param depth - Maximum depth in each direction (default: 2)
  1686. * @returns Subgraph containing the call graph
  1687. */
  1688. getCallGraph(nodeId: string, depth: number = 2): Subgraph {
  1689. return this.traverser.getCallGraph(nodeId, depth);
  1690. }
  1691. /**
  1692. * Get the type hierarchy for a class/interface
  1693. *
  1694. * Returns both ancestors (types this extends/implements) and
  1695. * descendants (types that extend/implement this).
  1696. *
  1697. * @param nodeId - ID of the class/interface node
  1698. * @returns Subgraph containing the type hierarchy
  1699. */
  1700. getTypeHierarchy(nodeId: string): Subgraph {
  1701. return this.traverser.getTypeHierarchy(nodeId);
  1702. }
  1703. /**
  1704. * Find all usages of a symbol
  1705. *
  1706. * Returns all nodes that reference the specified symbol through
  1707. * any edge type (calls, references, type_of, etc.).
  1708. *
  1709. * @param nodeId - ID of the symbol node
  1710. * @returns Array of nodes and edges that reference this symbol
  1711. */
  1712. findUsages(nodeId: string): Array<{ node: Node; edge: Edge }> {
  1713. return this.traverser.findUsages(nodeId);
  1714. }
  1715. /**
  1716. * Get callers of a function/method
  1717. *
  1718. * @param nodeId - ID of the function/method node
  1719. * @param maxDepth - Maximum depth to traverse (default: 1)
  1720. * @returns Array of nodes that call this function
  1721. */
  1722. getCallers(nodeId: string, maxDepth: number = 1): Array<{ node: Node; edge: Edge }> {
  1723. return this.traverser.getCallers(nodeId, maxDepth);
  1724. }
  1725. /**
  1726. * Get callees of a function/method
  1727. *
  1728. * @param nodeId - ID of the function/method node
  1729. * @param maxDepth - Maximum depth to traverse (default: 1)
  1730. * @returns Array of nodes called by this function
  1731. */
  1732. getCallees(nodeId: string, maxDepth: number = 1): Array<{ node: Node; edge: Edge }> {
  1733. return this.traverser.getCallees(nodeId, maxDepth);
  1734. }
  1735. /**
  1736. * Calculate the impact radius of a node
  1737. *
  1738. * Returns all nodes that could be affected by changes to this node.
  1739. *
  1740. * @param nodeId - ID of the node
  1741. * @param maxDepth - Maximum depth to traverse (default: 3)
  1742. * @returns Subgraph containing potentially impacted nodes
  1743. */
  1744. getImpactRadius(nodeId: string, maxDepth: number = 3): Subgraph {
  1745. return this.traverser.getImpactRadius(nodeId, maxDepth);
  1746. }
  1747. /**
  1748. * Find the shortest path between two nodes
  1749. *
  1750. * @param fromId - Starting node ID
  1751. * @param toId - Target node ID
  1752. * @param edgeKinds - Edge types to consider (all if empty)
  1753. * @returns Array of nodes and edges forming the path, or null if no path exists
  1754. */
  1755. findPath(
  1756. fromId: string,
  1757. toId: string,
  1758. edgeKinds?: Edge['kind'][]
  1759. ): Array<{ node: Node; edge: Edge | null }> | null {
  1760. return this.traverser.findPath(fromId, toId, edgeKinds);
  1761. }
  1762. /**
  1763. * Get ancestors of a node in the containment hierarchy
  1764. *
  1765. * @param nodeId - ID of the node
  1766. * @returns Array of ancestor nodes from immediate parent to root
  1767. */
  1768. getAncestors(nodeId: string): Node[] {
  1769. return this.traverser.getAncestors(nodeId);
  1770. }
  1771. /**
  1772. * Get immediate children of a node
  1773. *
  1774. * @param nodeId - ID of the node
  1775. * @returns Array of child nodes
  1776. */
  1777. getChildren(nodeId: string): Node[] {
  1778. return this.traverser.getChildren(nodeId);
  1779. }
  1780. /**
  1781. * Get dependencies of a file
  1782. *
  1783. * @param filePath - Path to the file
  1784. * @returns Array of file paths this file depends on
  1785. */
  1786. getFileDependencies(filePath: string): string[] {
  1787. return this.graphManager.getFileDependencies(filePath);
  1788. }
  1789. /**
  1790. * Get dependents of a file
  1791. *
  1792. * @param filePath - Path to the file
  1793. * @returns Array of file paths that depend on this file
  1794. */
  1795. getFileDependents(filePath: string): string[] {
  1796. return this.graphManager.getFileDependents(filePath);
  1797. }
  1798. /**
  1799. * Find circular dependencies in the codebase
  1800. *
  1801. * @returns Array of cycles, each cycle is an array of file paths
  1802. */
  1803. findCircularDependencies(): string[][] {
  1804. return this.graphManager.findCircularDependencies();
  1805. }
  1806. /**
  1807. * Find dead code (unreferenced symbols)
  1808. *
  1809. * @param kinds - Node kinds to check (default: functions, methods, classes)
  1810. * @returns Array of unreferenced nodes
  1811. */
  1812. findDeadCode(kinds?: Node['kind'][]): Node[] {
  1813. return this.graphManager.findDeadCode(kinds);
  1814. }
  1815. /**
  1816. * Get complexity metrics for a node
  1817. *
  1818. * @param nodeId - ID of the node
  1819. * @returns Object containing various complexity metrics
  1820. */
  1821. getNodeMetrics(nodeId: string): {
  1822. incomingEdgeCount: number;
  1823. outgoingEdgeCount: number;
  1824. callCount: number;
  1825. callerCount: number;
  1826. childCount: number;
  1827. depth: number;
  1828. } {
  1829. return this.graphManager.getNodeMetrics(nodeId);
  1830. }
  1831. // ===========================================================================
  1832. // Context Building
  1833. // ===========================================================================
  1834. /**
  1835. * Get the source code for a node
  1836. *
  1837. * Reads the file and extracts the code between startLine and endLine.
  1838. *
  1839. * @param nodeId - ID of the node
  1840. * @returns Code string or null if not found
  1841. */
  1842. async getCode(nodeId: string): Promise<string | null> {
  1843. return this.contextBuilder.getCode(nodeId);
  1844. }
  1845. /**
  1846. * Find relevant subgraph for a query
  1847. *
  1848. * Combines semantic search with graph traversal to find the most
  1849. * relevant nodes and their relationships for a given query.
  1850. *
  1851. * @param query - Natural language query describing the task
  1852. * @param options - Search and traversal options
  1853. * @returns Subgraph of relevant nodes and edges
  1854. */
  1855. async findRelevantContext(
  1856. query: string,
  1857. options?: FindRelevantContextOptions
  1858. ): Promise<Subgraph> {
  1859. // Segment-vocab supplement: FTS keeps camelCase names as single tokens,
  1860. // so a word-level query ("auto-scroll to bottom") can never reach
  1861. // `pinFeedIfNearBottom` through search alone. Resolve the query's words
  1862. // against name_segment_vocab (same precision rules as the prompt hook:
  1863. // co-occurrence, else rare singles, verified against live nodes) and hand
  1864. // the names down as dampened exact-name seeds. Callers that pass their
  1865. // own seedNames keep them; failures degrade to no supplement.
  1866. let seedNames = options?.seedNames;
  1867. if (seedNames === undefined) {
  1868. try {
  1869. seedNames = this.getSegmentMatches(extractSegmentSearchWords(query), 8)
  1870. .map((m) => m.name);
  1871. } catch {
  1872. seedNames = [];
  1873. }
  1874. }
  1875. return this.contextBuilder.findRelevantContext(query, { ...options, seedNames });
  1876. }
  1877. /**
  1878. * Build context for a task
  1879. *
  1880. * Creates comprehensive context by:
  1881. * 1. Running FTS search to find entry points
  1882. * 2. Expanding the graph around entry points
  1883. * 3. Extracting code blocks for key nodes
  1884. * 4. Formatting output for Claude
  1885. *
  1886. * @param input - Task description (string or {title, description})
  1887. * @param options - Build options (maxNodes, includeCode, format, etc.)
  1888. * @returns TaskContext object or formatted string (markdown/JSON)
  1889. */
  1890. async buildContext(
  1891. input: TaskInput,
  1892. options?: BuildContextOptions
  1893. ): Promise<TaskContext | string> {
  1894. return this.contextBuilder.buildContext(input, options);
  1895. }
  1896. // ===========================================================================
  1897. // Database Management
  1898. // ===========================================================================
  1899. /**
  1900. * Optimize the database (vacuum and analyze)
  1901. */
  1902. optimize(): void {
  1903. this.db.optimize();
  1904. }
  1905. /**
  1906. * Clear all data from the graph
  1907. */
  1908. clear(): void {
  1909. this.queries.clear();
  1910. }
  1911. /**
  1912. * Alias for close() for backwards compatibility.
  1913. * @deprecated Use close() instead
  1914. */
  1915. destroy(): void {
  1916. this.close();
  1917. }
  1918. /**
  1919. * Completely remove CodeGraph from the project.
  1920. * This closes the database and deletes the .CodeGraph directory.
  1921. *
  1922. * WARNING: This permanently deletes all CodeGraph data for the project.
  1923. */
  1924. uninitialize(): void {
  1925. this.close();
  1926. removeDirectory(this.projectRoot);
  1927. }
  1928. }
  1929. // Default export
  1930. export default CodeGraph;