/** * Database Queries * * Prepared statements for CRUD operations on the knowledge graph. */ import { SqliteDatabase, SqliteStatement } from './sqlite-adapter'; import { Node, Edge, FileRecord, UnresolvedReference, NodeKind, EdgeKind, Language, GraphStats, SearchOptions, SearchResult, } from '../types'; import { safeJsonParse } from '../utils'; import { kindBonus, nameMatchBonus, scorePathRelevance } from '../search/query-utils'; import { parseQuery, boundedEditDistance } from '../search/query-parser'; import { isGeneratedFile } from '../extraction/generated-detection'; import { splitIdentifierSegments } from '../search/identifier-segments'; /** * Files that should not be candidates for "dominant file" detection: test/spec * files and tool-generated files. Generated files (`*.pb.go`, `*.pulsar.go`, * mock outputs, …) often have huge in-file edge counts that dwarf the real * source — etcd's `rpc.pb.go` has 4× the in-file edges of `server.go`. * * Path patterns plus, when the caller passes the indexed set, files whose * HEADER declares them generated — a `payroll.go` full of generated CRUD has * exactly the same edge-density problem as `rpc.pb.go` and nothing in its name * to catch it (#1500). */ function isLowValueFile(filePath: string, generated?: ReadonlySet): boolean { if (generated?.has(filePath)) return true; const lp = filePath.toLowerCase(); return ( /(?:^|\/)(tests?|__tests?__|spec)\//.test(lp) || /_test\.go$/.test(lp) || /(?:^|\/)test_[^/]+\.py$/.test(lp) || /_test\.py$/.test(lp) || /_spec\.rb$/.test(lp) || /_test\.rb$/.test(lp) || /\.(test|spec)\.[jt]sx?$/.test(lp) || /(test|spec|tests)\.(java|kt|scala)$/.test(lp) || /(tests?|spec)\.cs$/.test(lp) || /tests?\.swift$/.test(lp) || /_test\.dart$/.test(lp) || isGeneratedFile(filePath) ); } const SQLITE_PARAM_CHUNK_SIZE = 500; /** * Database row types (snake_case from SQLite) */ interface NodeRow { id: string; kind: string; name: string; qualified_name: string; file_path: string; language: string; start_line: number; end_line: number; start_column: number; end_column: number; docstring: string | null; signature: string | null; visibility: string | null; is_exported: number; is_async: number; is_static: number; is_abstract: number; decorators: string | null; type_parameters: string | null; return_type: string | null; updated_at: number; } interface EdgeRow { id: number; source: string; target: string; kind: string; metadata: string | null; line: number | null; col: number | null; provenance: string | null; } interface FileRow { path: string; content_hash: string; language: string; size: number; modified_at: number; indexed_at: number; node_count: number; errors: string | null; /** Absent on pre-v9 rows read through a stale prepared statement. */ generated?: number | null; } interface UnresolvedRefRow { id: number; from_node_id: string; reference_name: string; reference_kind: string; line: number; col: number; candidates: string | null; file_path: string; language: string; status: string; name_tail: string; } /** * Last segment of a (possibly dotted/qualified) reference name — the part a * new symbol's plain node name could match: 'util.greet' → 'greet', * 'mod::fn' → 'fn', 'greet' → 'greet'. Written to unresolved_refs.name_tail * when a ref is marked failed, so the #1240 retry lookup can match dotted * refs against newly-added node names. */ function referenceNameTail(referenceName: string): string { const idx = Math.max(referenceName.lastIndexOf('.'), referenceName.lastIndexOf(':')); return idx >= 0 ? referenceName.slice(idx + 1) : referenceName; } /** * Convert database row to Node object */ function rowToNode(row: NodeRow): Node { return { id: row.id, kind: row.kind as NodeKind, name: row.name, qualifiedName: row.qualified_name, filePath: row.file_path, language: row.language as Language, startLine: row.start_line, endLine: row.end_line, startColumn: row.start_column, endColumn: row.end_column, docstring: row.docstring ?? undefined, signature: row.signature ?? undefined, visibility: row.visibility as Node['visibility'], isExported: row.is_exported === 1, isAsync: row.is_async === 1, isStatic: row.is_static === 1, isAbstract: row.is_abstract === 1, decorators: row.decorators ? safeJsonParse(row.decorators, undefined) : undefined, typeParameters: row.type_parameters ? safeJsonParse(row.type_parameters, undefined) : undefined, returnType: row.return_type ?? undefined, updatedAt: row.updated_at, }; } /** * Convert database row to Edge object */ function rowToEdge(row: EdgeRow): Edge { return { source: row.source, target: row.target, kind: row.kind as EdgeKind, metadata: row.metadata ? safeJsonParse(row.metadata, undefined) : undefined, line: row.line ?? undefined, column: row.col ?? undefined, provenance: row.provenance as Edge['provenance'], }; } /** * Convert database row to FileRecord object */ function rowToFileRecord(row: FileRow): FileRecord { return { path: row.path, contentHash: row.content_hash, language: row.language as Language, size: row.size, modifiedAt: row.modified_at, indexedAt: row.indexed_at, nodeCount: row.node_count, errors: row.errors ? safeJsonParse(row.errors, undefined) : undefined, generated: row.generated === 1, }; } /** * Query builder for the knowledge graph database */ export class QueryBuilder { private db: SqliteDatabase; // Project-name tokens (go.mod / package.json / repo dir), normalized. A query // word matching one is dropped from path-relevance scoring — it names the // whole project, not a symbol, so it carries no discriminative signal (#720). // Set once by the CodeGraph instance; empty by default (no down-weighting). private projectNameTokens: Set = new Set(); // Node cache for frequently accessed nodes (LRU-style, max 1000 entries) private nodeCache: Map = new Map(); private readonly maxCacheSize = 1000; // Prepared statements (lazily initialized) private stmts: { insertNode?: SqliteStatement; updateNode?: SqliteStatement; deleteNode?: SqliteStatement; deleteNodesByFile?: SqliteStatement; getNodeById?: SqliteStatement; getNodesByFile?: SqliteStatement; getNodesByKind?: SqliteStatement; insertEdge?: SqliteStatement; upsertFile?: SqliteStatement; deleteEdgesBySource?: SqliteStatement; deleteEdgesByTarget?: SqliteStatement; getEdgesBySource?: SqliteStatement; getEdgesByTarget?: SqliteStatement; insertFile?: SqliteStatement; updateFile?: SqliteStatement; deleteFile?: SqliteStatement; getFileByPath?: SqliteStatement; getAllFiles?: SqliteStatement; insertUnresolved?: SqliteStatement; deleteUnresolvedByNode?: SqliteStatement; getUnresolvedByName?: SqliteStatement; getNodesByName?: SqliteStatement; getNodesByNamePrefix?: SqliteStatement; getNodesByQualifiedNameExact?: SqliteStatement; getNodesByLowerName?: SqliteStatement; getUnresolvedCount?: SqliteStatement; getUnresolvedBatch?: SqliteStatement; getUnresolvedBatchAfter?: SqliteStatement; deleteRefsByRowIdsFull?: SqliteStatement; getAllFilePaths?: SqliteStatement; getAllNodeNames?: SqliteStatement; getDominantFile?: SqliteStatement; getTopRouteFile?: SqliteStatement; getRoutingManifest?: SqliteStatement; insertNameSegment?: SqliteStatement; } = {}; // Names whose segments were already written this session — skips re-splitting // and re-inserting for the same-named nodes that repeat across files ("get", // "render", …). Purely a write-path fast path; INSERT OR IGNORE is the // correctness backstop. Bounded so a pathological repo can't grow it forever. private segmentedNames: Set = new Set(); private static readonly MAX_SEGMENTED_NAMES = 65536; // Multi-row INSERT statements, cached per (statement kind × row count). The // bulk write path decomposes N rows into a few fixed batch sizes so each // size's statement is prepared once and reused — one .run() binds a whole // chunk instead of one row, which is where the per-call overhead lives. // Row order within and across chunks is the input order, so rowid assignment // (and therefore resolution's insertion-order disambiguation) is identical // to the one-row-per-run path. private batchStmts: Map = new Map(); private static readonly BATCH_SIZES: readonly number[] = [128, 32, 8, 1]; /** * Run `rows` through a multi-row `INSERT` built as `head + (tuple,)*n`, * decomposed greedily into the cached batch sizes. Preserves row order. */ private runBatched(kind: string, head: string, tuple: string, rows: unknown[][]): void { if (rows.length === 0) return; let i = 0; for (const size of QueryBuilder.BATCH_SIZES) { while (rows.length - i >= size) { const key = `${kind}:${size}`; let stmt = this.batchStmts.get(key); if (!stmt) { stmt = this.db.prepare(head + new Array(size).fill(tuple).join(',')); this.batchStmts.set(key, stmt); } if (size === 1) { stmt.run(...rows[i]!); } else { const params: unknown[] = []; for (let r = 0; r < size; r++) { const row = rows[i + r]!; for (let c = 0; c < row.length; c++) params.push(row[c]); } stmt.run(...params); } i += size; } } } constructor(db: SqliteDatabase) { this.db = db; } /** * Swap the underlying connection in place. Used by pool workers' * connection recycling (plan §7a.6, writes-under-readers): a long-lived * read connection pins WAL checkpoint progress, and the deep WAL that * accumulates behind it taxes every main-thread B-tree page operation * (deletes measured 42.6s → 118.8s from 0 to 4 attached readers on * identical hardware). Workers therefore close and reopen their read-only * connection at the pool-idle boundary; everything above the connection — * this QueryBuilder, the resolver and its warm caches — survives, and only * connection-derived state (prepared statements) resets, re-preparing * lazily on next use. */ rebind(db: SqliteDatabase): void { this.db = db; this.stmts = {}; this.batchStmts.clear(); } /** Set the normalized project-name tokens used to down-weight non-discriminative * query words in path scoring (#720). Called once when the project opens. */ setProjectNameTokens(tokens: Set): void { this.projectNameTokens = tokens; } /** The normalized project-name tokens (#720); empty if none were derived. */ getProjectNameTokens(): Set { return this.projectNameTokens; } // =========================================================================== // Node Operations // =========================================================================== /** * Insert a new node */ insertNode(node: Node): void { if (!this.stmts.insertNode) { this.stmts.insertNode = this.db.prepare(` INSERT OR REPLACE INTO nodes ( id, kind, name, qualified_name, file_path, language, start_line, end_line, start_column, end_column, docstring, signature, visibility, is_exported, is_async, is_static, is_abstract, decorators, type_parameters, return_type, updated_at ) VALUES ( @id, @kind, @name, @qualifiedName, @filePath, @language, @startLine, @endLine, @startColumn, @endColumn, @docstring, @signature, @visibility, @isExported, @isAsync, @isStatic, @isAbstract, @decorators, @typeParameters, @returnType, @updatedAt ) `); } // Validate required fields to prevent SQLite bind errors if (!node.id || !node.kind || !node.name || !node.filePath || !node.language) { console.error('[CodeGraph] Skipping node with missing required fields:', { id: node.id, kind: node.kind, name: node.name, filePath: node.filePath, language: node.language, }); return; } // INSERT OR REPLACE may overwrite a node we have cached. Drop the // stale entry so the next getNodeById sees the new row, not the old // one (matches the cache-invalidation pattern used by updateNode and // deleteNode below). this.nodeCache.delete(node.id); this.stmts.insertNode.run({ id: node.id, kind: node.kind, name: node.name, qualifiedName: node.qualifiedName ?? node.name, filePath: node.filePath, language: node.language, startLine: node.startLine ?? 0, endLine: node.endLine ?? 0, startColumn: node.startColumn ?? 0, endColumn: node.endColumn ?? 0, docstring: node.docstring ?? null, signature: node.signature ?? null, visibility: node.visibility ?? null, isExported: node.isExported ? 1 : 0, isAsync: node.isAsync ? 1 : 0, isStatic: node.isStatic ? 1 : 0, isAbstract: node.isAbstract ? 1 : 0, decorators: node.decorators ? JSON.stringify(node.decorators) : null, typeParameters: node.typeParameters ? JSON.stringify(node.typeParameters) : null, returnType: node.returnType ?? null, updatedAt: node.updatedAt ?? Date.now(), }); // Segment vocabulary rides the same write path (and transaction) so it can // never drift ahead of the nodes it describes. Deletes intentionally leave // orphans behind — vocab rows are proposals re-verified against nodes // before use, and a full index clears the table at its start. File nodes // are excluded: a file's basename duplicates the symbols inside it // (state-machine.ts / OrderStateMachine), which double-counts every // concept and defeats the singleton-vs-cluster rarity statistics. Import // nodes are excluded too (#1144): they're named after module specifiers // ("external-unindexed-pkg", "./utils/helpers"), not symbols — an // import-only name can never be surfaced (getSegmentMatches requires a // real definition), so its rows would only inflate the rarity statistics. if (this.isSegmentableKind(node.kind)) this.insertNameSegments(node.name); } /** Which node kinds contribute their name to the segment vocabulary — the * single gate shared by insertNode, updateNode, and the rebuild page query * (getDistinctNodeNames), so the write paths can't drift apart. */ private isSegmentableKind(kind: string): boolean { return kind !== 'file' && kind !== 'import'; } /** Write `name`'s segments into name_segment_vocab (idempotent). */ private insertNameSegments(name: string): void { const rows: unknown[][] = []; this.collectNameSegmentRows(name, rows); this.runBatched( 'insertNameSegments', 'INSERT OR IGNORE INTO name_segment_vocab (segment, name) VALUES ', '(?,?)', rows ); } /** * Insert multiple nodes in a transaction */ insertNodes(nodes: Node[]): void { this.db.transaction(() => { // Bulk path: same semantics as insertNode() per row (validation, cache // invalidation, segment vocab), but bound as multi-row INSERTs — the // per-.run() call overhead dominates the store phase on full indexes. const rows: unknown[][] = []; const segmentRows: unknown[][] = []; for (const node of nodes) { if (!node.id || !node.kind || !node.name || !node.filePath || !node.language) { console.error('[CodeGraph] Skipping node with missing required fields:', { id: node.id, kind: node.kind, name: node.name, filePath: node.filePath, language: node.language, }); continue; } this.nodeCache.delete(node.id); rows.push([ node.id, node.kind, node.name, node.qualifiedName ?? node.name, node.filePath, node.language, node.startLine ?? 0, node.endLine ?? 0, node.startColumn ?? 0, node.endColumn ?? 0, node.docstring ?? null, node.signature ?? null, node.visibility ?? null, node.isExported ? 1 : 0, node.isAsync ? 1 : 0, node.isStatic ? 1 : 0, node.isAbstract ? 1 : 0, node.decorators ? JSON.stringify(node.decorators) : null, node.typeParameters ? JSON.stringify(node.typeParameters) : null, node.returnType ?? null, node.updatedAt ?? Date.now(), ]); if (this.isSegmentableKind(node.kind)) this.collectNameSegmentRows(node.name, segmentRows); } this.runBatched( 'insertNodes', `INSERT OR REPLACE INTO nodes ( id, kind, name, qualified_name, file_path, language, start_line, end_line, start_column, end_column, docstring, signature, visibility, is_exported, is_async, is_static, is_abstract, decorators, type_parameters, return_type, updated_at ) VALUES `, '(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', rows ); this.runBatched( 'insertNameSegments', 'INSERT OR IGNORE INTO name_segment_vocab (segment, name) VALUES ', '(?,?)', segmentRows ); })(); } /** * Store one file's whole extraction bundle — nodes, edges, unresolved refs, * and the file record — in a SINGLE transaction. The bulk-index path calls * this once per file instead of opening one transaction per table (#1015 * file-order commit discipline is unchanged: callers still invoke it in file * order, and row order within is input order). * * Edges MUST already be endpoint-filtered by the caller (the store path * filters to the file's own inserted node ids), so the per-file existence * SELECT that insertEdges() pays is skipped here. */ storeFileBundle(bundle: { nodes: Node[]; edges: Edge[]; refs: UnresolvedReference[]; file: FileRecord; }): void { this.db.transaction(() => { this.insertNodes(bundle.nodes); if (bundle.edges.length > 0) { const rows: unknown[][] = []; for (const edge of bundle.edges) { rows.push([ edge.source, edge.target, edge.kind, edge.metadata ? JSON.stringify(edge.metadata) : null, edge.line ?? null, edge.column ?? null, edge.provenance ?? null, ]); } this.runBatched( 'insertEdges', 'INSERT OR IGNORE INTO edges (source, target, kind, metadata, line, col, provenance) VALUES ', '(?,?,?,?,?,?,?)', rows ); } if (bundle.refs.length > 0) this.insertUnresolvedRefsBatch(bundle.refs); this.upsertFile(bundle.file); })(); } /** * Collect (segment, name) rows for a name, honouring the same session-dedupe * semantics as insertNameSegments(). Shared by the bulk write paths. */ private collectNameSegmentRows(name: string, out: unknown[][]): void { if (this.segmentedNames.has(name)) return; if (this.segmentedNames.size >= QueryBuilder.MAX_SEGMENTED_NAMES) this.segmentedNames.clear(); this.segmentedNames.add(name); for (const segment of splitIdentifierSegments(name)) out.push([segment, name]); } /** * Update an existing node */ updateNode(node: Node): void { if (!this.stmts.updateNode) { this.stmts.updateNode = this.db.prepare(` UPDATE nodes SET kind = @kind, name = @name, qualified_name = @qualifiedName, file_path = @filePath, language = @language, start_line = @startLine, end_line = @endLine, start_column = @startColumn, end_column = @endColumn, docstring = @docstring, signature = @signature, visibility = @visibility, is_exported = @isExported, is_async = @isAsync, is_static = @isStatic, is_abstract = @isAbstract, decorators = @decorators, type_parameters = @typeParameters, return_type = @returnType, updated_at = @updatedAt WHERE id = @id `); } // Invalidate cache before update this.nodeCache.delete(node.id); // Validate required fields if (!node.id || !node.kind || !node.name || !node.filePath || !node.language) { console.error('[CodeGraph] Skipping node update with missing required fields:', node.id); return; } this.stmts.updateNode.run({ id: node.id, kind: node.kind, name: node.name, qualifiedName: node.qualifiedName ?? node.name, filePath: node.filePath, language: node.language, startLine: node.startLine ?? 0, endLine: node.endLine ?? 0, startColumn: node.startColumn ?? 0, endColumn: node.endColumn ?? 0, docstring: node.docstring ?? null, signature: node.signature ?? null, visibility: node.visibility ?? null, isExported: node.isExported ? 1 : 0, isAsync: node.isAsync ? 1 : 0, isStatic: node.isStatic ? 1 : 0, isAbstract: node.isAbstract ? 1 : 0, decorators: node.decorators ? JSON.stringify(node.decorators) : null, typeParameters: node.typeParameters ? JSON.stringify(node.typeParameters) : null, returnType: node.returnType ?? null, updatedAt: node.updatedAt ?? Date.now(), }); // updateNode is a second real write path to `nodes` — framework // post-extract passes rewrite names through it (NestJS route prefixing), // and a renamed node's new name must reach the segment vocabulary just // like an inserted one's (#1141). Without this the rename left the new // name permanently unsearchable: the old name's rows became honest-gate // orphans and the only backfill is gated on the vocab being EMPTY. // insertNameSegments is idempotent (in-memory set + INSERT OR IGNORE), // so no name-changed check is needed. if (this.isSegmentableKind(node.kind)) this.insertNameSegments(node.name); } /** * Delete a node by ID */ deleteNode(id: string): void { if (!this.stmts.deleteNode) { this.stmts.deleteNode = this.db.prepare('DELETE FROM nodes WHERE id = ?'); } // Invalidate cache this.nodeCache.delete(id); this.stmts.deleteNode.run(id); } /** * Delete all nodes for a file */ deleteNodesByFile(filePath: string): void { if (!this.stmts.deleteNodesByFile) { this.stmts.deleteNodesByFile = this.db.prepare('DELETE FROM nodes WHERE file_path = ?'); } // Invalidate cache for nodes in this file for (const [id, node] of this.nodeCache) { if (node.filePath === filePath) { this.nodeCache.delete(id); } } this.stmts.deleteNodesByFile.run(filePath); } // =========================================================================== // Name-segment vocabulary (prompt-hook graph-derived gate) // =========================================================================== /** Wipe the segment vocabulary. A full index calls this at its start; the * node write path repopulates it as files (re-)index, so the end state is * exactly the current names with no orphan rows. */ clearNameSegmentVocab(): void { this.db.exec('DELETE FROM name_segment_vocab'); this.segmentedNames.clear(); } /** True when the vocab has no rows — an index built before the table existed. * `sync` uses this to heal such databases (see rebuildNameSegmentVocabFrom). */ isNameSegmentVocabEmpty(): boolean { const row = this.db.prepare('SELECT 1 FROM name_segment_vocab LIMIT 1').get(); return row === undefined; } /** One page of distinct segmentable node names, for batched vocab rebuilds * (file basenames and import specifiers are excluded from the vocab — see * insertNode). */ getDistinctNodeNames(limit: number, offset: number): string[] { const rows = this.db .prepare("SELECT DISTINCT name FROM nodes WHERE kind NOT IN ('file', 'import') ORDER BY name LIMIT ? OFFSET ?") .all(limit, offset) as Array<{ name: string }>; return rows.map((r) => r.name); } /** Insert segments for a batch of names in one transaction (vocab heal path). */ insertNameSegmentsBatch(names: string[]): void { this.db.transaction(() => { const rows: unknown[][] = []; for (const name of names) this.collectNameSegmentRows(name, rows); this.runBatched( 'insertNameSegments', 'INSERT OR IGNORE INTO name_segment_vocab (segment, name) VALUES ', '(?,?)', rows ); })(); } /** * Names whose segments cover at least `minWords` distinct PROMPT WORDS — * the co-occurrence probe behind the prompt hook's medium tier: the words * "state" and "machine" both being segments of `OrderStateMachine` is strong * evidence the prompt names that symbol in prose. Ordered by coverage. * * Takes (segment variant → original word) pairs and folds variants back to * their word INSIDE the SQL: a name matching both `service` and `services` * counts ONE word, not two. Counting raw variants let plural-variant pairs * of a single word tie with genuine two-word matches and — because ORDER * BY/LIMIT run here, before any JS-side re-check — crowd a real match past * the LIMIT on vocab-heavy repos (#1146). */ getSegmentCoOccurrence( variants: Array<{ segment: string; word: string }>, minWords: number, limit: number, ): Array<{ name: string; matches: number }> { if (variants.length === 0) return []; const placeholders = variants.map(() => '?').join(', '); const whens = variants.map(() => 'WHEN ? THEN ?').join(' '); const rows = this.db .prepare( `SELECT name, COUNT(DISTINCT CASE segment ${whens} END) AS matches FROM name_segment_vocab WHERE segment IN (${placeholders}) GROUP BY name HAVING matches >= ? ORDER BY matches DESC, length(name) ASC LIMIT ?`, ) .all( ...variants.flatMap((v) => [v.segment, v.word]), ...variants.map((v) => v.segment), minWords, limit, ) as Array<{ name: string; matches: number }>; return rows; } /** How many distinct names each segment appears in — the rarity signal that * separates a discriminative word ("checkout") from a ubiquitous one ("state"). */ getSegmentNameCounts(segments: string[]): Map { if (segments.length === 0) return new Map(); const placeholders = segments.map(() => '?').join(', '); const rows = this.db .prepare( `SELECT segment, COUNT(*) AS n FROM name_segment_vocab WHERE segment IN (${placeholders}) GROUP BY segment`, ) .all(...segments) as Array<{ segment: string; n: number }>; return new Map(rows.map((r) => [r.segment, r.n])); } /** Names containing the given segment (rare-single-word tier). */ getNamesForSegment(segment: string, limit: number): string[] { const rows = this.db .prepare('SELECT name FROM name_segment_vocab WHERE segment = ? ORDER BY length(name) ASC LIMIT ?') .all(segment, limit) as Array<{ name: string }>; return rows.map((r) => r.name); } /** * Get a node by ID */ getNodeById(id: string): Node | null { // Check cache first if (this.nodeCache.has(id)) { const cached = this.nodeCache.get(id)!; // Move to end to implement LRU (delete and re-add) this.nodeCache.delete(id); this.nodeCache.set(id, cached); return cached; } if (!this.stmts.getNodeById) { this.stmts.getNodeById = this.db.prepare('SELECT * FROM nodes WHERE id = ?'); } const row = this.stmts.getNodeById.get(id) as NodeRow | undefined; if (!row) { return null; } const node = rowToNode(row); this.cacheNode(node); return node; } /** * Batch lookup: fetch many nodes by ID in a single SQL round-trip. * * Replaces the N+1 pattern in graph traversal where every edge would * trigger its own `getNodeById` call. For a function with 50 callers * this collapses 50 point reads into one IN-list query (~10-50x * faster end-to-end). * * Returns a Map keyed by id so callers can preserve their own ordering * (typically the order edges were returned from the graph). Missing IDs * are simply absent from the map. * * Cache-aware: ids already in the LRU cache are served from memory and * the SQL query only touches the misses. */ getNodesByIds(ids: readonly string[]): Map { const out = new Map(); if (ids.length === 0) return out; // Serve cache hits first; build the miss list for SQL. const misses: string[] = []; for (const id of ids) { const cached = this.nodeCache.get(id); if (cached !== undefined) { // LRU touch this.nodeCache.delete(id); this.nodeCache.set(id, cached); out.set(id, cached); } else { misses.push(id); } } if (misses.length === 0) return out; // Chunk under SQLite's parameter limit (default 999, raised to 32766 // in better-sqlite3 builds — chunk at 500 for safety across both // backends and to keep the query plan simple). for (let i = 0; i < misses.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = misses.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); const rows = this.db .prepare(`SELECT * FROM nodes WHERE id IN (${placeholders})`) .all(...chunk) as NodeRow[]; for (const row of rows) { const node = rowToNode(row); out.set(node.id, node); this.cacheNode(node); } } return out; } private getExistingNodeIds(ids: readonly string[]): Set { const out = new Set(); if (ids.length === 0) return out; const uniqueIds = [...new Set(ids)]; for (let i = 0; i < uniqueIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = uniqueIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); const rows = this.db .prepare(`SELECT id FROM nodes WHERE id IN (${placeholders})`) .all(...chunk) as { id: string }[]; for (const row of rows) { out.add(row.id); } } return out; } /** * Add a node to the cache, evicting oldest if needed */ private cacheNode(node: Node): void { if (this.nodeCache.size >= this.maxCacheSize) { // Evict oldest (first) entry const firstKey = this.nodeCache.keys().next().value; if (firstKey) { this.nodeCache.delete(firstKey); } } this.nodeCache.set(node.id, node); } /** * Clear the node cache */ clearCache(): void { this.nodeCache.clear(); } /** * Get all nodes in a file */ getNodesByFile(filePath: string): Node[] { if (!this.stmts.getNodesByFile) { this.stmts.getNodesByFile = this.db.prepare( 'SELECT * FROM nodes WHERE file_path = ? ORDER BY start_line' ); } const rows = this.stmts.getNodesByFile.all(filePath) as NodeRow[]; return rows.map(rowToNode); } /** * Find the file that holds the densest concentration of the project's * internal call graph — the "core" file. Used by context-builder to * boost ranking of symbols in that file's directory (so e.g. sinatra * queries surface `lib/sinatra/base.rb`'s `route!` instead of * `sinatra-contrib/lib/sinatra/multi_route.rb`'s `route` extension). * * Returns null if no file has a meaningful concentration (e.g. spread * evenly across many files, or empty index). * * "Internal" = source and target are in the same file. Cross-file * edges aren't useful here — they don't tell us which file is the * functional center. * * Excludes test/spec files from candidacy via path-pattern. The agent's * typical question is "how does X work", not "how is X tested", so * boosting a test file's directory would be a misfire. */ getDominantFile(): { filePath: string; edgeCount: number; nextEdgeCount: number } | null { if (!this.stmts.getDominantFile) { // Pull top 20 candidates; we then filter out test/generated files // in code (regex-grade matching that SQL LIKE can't express). The // generated-file filter is critical — without it, etcd's // `api/etcdserverpb/rpc.pb.go` (1916 in-file edges, generated // protobuf stub) outranks the real `server/etcdserver/server.go` // (470 edges) by 4×, and the boost would push the agent toward // generated code. this.stmts.getDominantFile = this.db.prepare(` SELECT n.file_path AS file_path, COUNT(*) AS edge_count FROM edges e JOIN nodes n ON e.source = n.id JOIN nodes m ON e.target = m.id WHERE n.file_path = m.file_path GROUP BY n.file_path ORDER BY edge_count DESC LIMIT 20 `); } const rows = this.stmts.getDominantFile.all() as Array<{ file_path: string; edge_count: number }>; const generated = this.getGeneratedPathsAmong(rows.map(r => r.file_path)); const filtered = rows.filter(r => !isLowValueFile(r.file_path, generated)); if (filtered.length === 0 || filtered[0]!.edge_count < 20) return null; return { filePath: filtered[0]!.file_path, edgeCount: filtered[0]!.edge_count, nextEdgeCount: filtered[1]?.edge_count ?? 0, }; } /** * Find the file that holds the densest concentration of the project's * `route` nodes (framework-emitted: Express/Gin/Flask/Rails/Drupal/etc.). * Used by handleContext on small repos to inline the project's routing * config when the agent's query is about request flow — eliminating the * "Glob + Read routes.rb" pattern that beats codegraph on tiny realworld * template repos. * * Excludes test/generated files from candidacy. Returns null if there * are fewer than 3 non-test routes total, or if no file holds at least * 30% of them (diffuse routing → no single answer file). */ getTopRouteFile(): { filePath: string; routeCount: number; totalRoutes: number } | null { if (!this.stmts.getTopRouteFile) { this.stmts.getTopRouteFile = this.db.prepare(` SELECT file_path, COUNT(*) AS cnt FROM nodes WHERE kind = 'route' GROUP BY file_path ORDER BY cnt DESC LIMIT 20 `); } const rows = this.stmts.getTopRouteFile.all() as Array<{ file_path: string; cnt: number }>; const generated = this.getGeneratedPathsAmong(rows.map(r => r.file_path)); const filtered = rows.filter(r => !isLowValueFile(r.file_path, generated)); if (filtered.length === 0) return null; const totalRoutes = filtered.reduce((sum, r) => sum + r.cnt, 0); const top = filtered[0]!; if (totalRoutes < 3 || top.cnt < 3) return null; if (top.cnt / totalRoutes < 0.30) return null; return { filePath: top.file_path, routeCount: top.cnt, totalRoutes }; } /** * Build a URL → handler manifest from the index. Each route node's * `references` edge points at the function/method that handles the * request. We join them in one pass; the agent gets the canonical * routing answer ("POST /users/login → AuthController#login") without * having to parse the framework's route DSL itself. * * Also returns the file with the most handler endpoints — used as the * "top handler file" to inline source for, so the agent has both the * mapping AND the handler implementations. */ getRoutingManifest(limit: number = 40): { entries: Array<{ url: string; handler: string; handlerFile: string; handlerLine: number; handlerKind: string }>; topHandlerFile: string | null; topHandlerFileCount: number; totalRoutes: number; } | null { if (!this.stmts.getRoutingManifest) { // Edge kind varies across framework resolvers: Spring/Rails/ // Laravel/Drupal emit `references`, Express emits `calls`. Accept // both — the semantic is the same (route → its handler). this.stmts.getRoutingManifest = this.db.prepare(` SELECT r.name AS url, h.name AS handler, h.file_path AS handler_file, h.start_line AS handler_line, h.kind AS handler_kind FROM nodes r JOIN edges e ON e.source = r.id JOIN nodes h ON e.target = h.id WHERE r.kind = 'route' AND e.kind IN ('references', 'calls') AND h.kind IN ('function', 'method', 'class') ORDER BY r.file_path, r.start_line LIMIT ? `); } const rows = this.stmts.getRoutingManifest.all(limit) as Array<{ url: string; handler: string; handler_file: string; handler_line: number; handler_kind: string; }>; // Drop test/generated handlers — same hygiene as elsewhere. const generated = this.getGeneratedPathsAmong(rows.map(r => r.handler_file)); const filtered = rows.filter(r => !isLowValueFile(r.handler_file, generated)); if (filtered.length < 3) return null; // Identify the file holding the most handlers (the "primary handler file"). const fileCounts = new Map(); for (const r of filtered) { fileCounts.set(r.handler_file, (fileCounts.get(r.handler_file) ?? 0) + 1); } let topHandlerFile: string | null = null; let topHandlerFileCount = 0; for (const [file, count] of fileCounts) { if (count > topHandlerFileCount) { topHandlerFile = file; topHandlerFileCount = count; } } return { entries: filtered.map(r => ({ url: r.url, handler: r.handler, handlerFile: r.handler_file, handlerLine: r.handler_line, handlerKind: r.handler_kind, })), topHandlerFile, topHandlerFileCount, totalRoutes: filtered.length, }; } /** * Get all nodes of a specific kind */ getNodesByKind(kind: NodeKind): Node[] { if (!this.stmts.getNodesByKind) { this.stmts.getNodesByKind = this.db.prepare('SELECT * FROM nodes WHERE kind = ?'); } const rows = this.stmts.getNodesByKind.all(kind) as NodeRow[]; return rows.map(rowToNode); } /** * Stream every node of a kind one at a time (lazy) instead of materializing * them all like {@link getNodesByKind}. For unbounded kinds (`function`, * `method`) on a symbol-dense project the full array is gigabytes; the * dynamic-edge synthesizers only scan-and-filter, so they iterate to keep * memory O(1) in the node count rather than O(nodes) (#610). */ *iterateNodesByKind(kind: NodeKind): IterableIterator { // Fresh statement per call (not a cached one): an iterator holds an open // cursor, so a shared statement would conflict across overlapping scans. const stmt = this.db.prepare('SELECT * FROM nodes WHERE kind = ?'); for (const row of stmt.iterate(kind)) { yield rowToNode(row as NodeRow); } } /** * Get all nodes in the database */ getAllNodes(): Node[] { const rows = this.db.prepare('SELECT * FROM nodes').all() as NodeRow[]; return rows.map(rowToNode); } /** * Stream nodes of one language whose `decorators` JSON array contains * `decorator`. The LIKE on the JSON text is a cheap index-free pre-filter * (a decorator name can appear as a substring of another), so callers must * still exact-check `node.decorators.includes(decorator)`. Exists so the * kotlin expect/actual synthesizer never materializes the whole node table * the way `getAllNodes().filter(...)` did — that array alone OOM'd Node's * default heap on a 2M-node graph (#1212). */ *iterateNodesByLanguageWithDecorator(language: Language, decorator: string): IterableIterator { // Fresh statement per call — an iterator holds an open cursor (see // iterateNodesByKind). const stmt = this.db.prepare( "SELECT * FROM nodes WHERE language = ? AND decorators LIKE '%' || ? || '%'" ); for (const row of stmt.iterate(language, `"${decorator}"`)) { yield rowToNode(row as NodeRow); } } /** * Distinct languages present in the files table. One indexed aggregate — * lets the dynamic-edge synthesizers skip passes for languages the project * doesn't contain at all (a Kotlin pass has no work on a pure-C repo), so * their cost is zero rather than a full-graph scan that finds nothing (#1212). */ getDistinctFileLanguages(): Set { const rows = this.db.prepare('SELECT DISTINCT language FROM files').all() as Array<{ language: string }>; return new Set(rows.map((r) => r.language)); } /** * Get nodes by exact name match (uses idx_nodes_name index). * * This is resolution's candidate list, and the ORDER BY is load-bearing for * index correctness, not cosmetic (CG-33). When a reference names a symbol * that several files define and nothing disambiguates them, resolution binds * to the first candidate — so without an ORDER BY the winner was decided by * rowid, i.e. by the order files happened to be WRITTEN. A full index writes * them in scan order; an incremental sync appends each file as it changes, so * the same tree resolved to different edges depending on how the index was * built, and a long-lived synced index drifted away from a rebuild of itself * (measured at 4.3% of distinct edges, mostly `calls`). * * `(file_path, start_line)` is a property of the CODE, so both paths now pick * the same candidate. The sort is paid once per distinct name per resolution * run — ReferenceResolver memoizes this in its nameCache — and the population * is capped by AMBIGUOUS_NAME_CEILING (#999). */ getNodesByName(name: string): Node[] { if (!this.stmts.getNodesByName) { this.stmts.getNodesByName = this.db.prepare( 'SELECT * FROM nodes WHERE name = ? ORDER BY file_path, start_line' ); } const rows = this.stmts.getNodesByName.all(name) as NodeRow[]; return rows.map(rowToNode); } /** * Nodes whose name starts with `prefix`, by index range scan (a LIKE would * skip idx_nodes_name under SQLite's default case-insensitive LIKE). */ getNodesByNamePrefix(prefix: string, limit = 20): Node[] { if (!this.stmts.getNodesByNamePrefix) { this.stmts.getNodesByNamePrefix = this.db.prepare( 'SELECT * FROM nodes WHERE name >= ? AND name < ? ORDER BY name LIMIT ?' ); } const rows = this.stmts.getNodesByNamePrefix.all(prefix, prefix + '￿', limit) as NodeRow[]; return rows.map(rowToNode); } /** * Get nodes by exact qualified name match (uses idx_nodes_qualified_name index) */ getNodesByQualifiedNameExact(qualifiedName: string): Node[] { if (!this.stmts.getNodesByQualifiedNameExact) { this.stmts.getNodesByQualifiedNameExact = this.db.prepare( 'SELECT * FROM nodes WHERE qualified_name = ?' ); } const rows = this.stmts.getNodesByQualifiedNameExact.all(qualifiedName) as NodeRow[]; return rows.map(rowToNode); } /** * Get nodes by lowercase name match (uses idx_nodes_lower_name expression index) */ getNodesByLowerName(lowerName: string): Node[] { if (!this.stmts.getNodesByLowerName) { this.stmts.getNodesByLowerName = this.db.prepare( 'SELECT * FROM nodes WHERE lower(name) = ?' ); } const rows = this.stmts.getNodesByLowerName.all(lowerName) as NodeRow[]; return rows.map(rowToNode); } /** * Search nodes by name using FTS with fallback to LIKE for better matching * * Search strategy: * 1. Try FTS5 prefix match (query*) for word-start matching * 2. If no results, try LIKE for substring matching (e.g., "signIn" finds "signInWithGoogle") * 3. Score results based on match quality */ searchNodes(query: string, options: SearchOptions = {}): SearchResult[] { const { limit = 100, offset = 0 } = options; // Parse field-qualified bits out of the raw query (kind:, lang:, // path:, name:). Anything not recognised stays in `text` and goes // to FTS unchanged. Filters compose with the SearchOptions arg — // both are applied (intersection-style). const parsed = parseQuery(query); const mergedKinds = parsed.kinds.length > 0 ? Array.from(new Set([...(options.kinds ?? []), ...parsed.kinds])) : options.kinds; const mergedLanguages = parsed.languages.length > 0 ? Array.from(new Set([...(options.languages ?? []), ...parsed.languages])) : options.languages; const pathFilters = parsed.pathFilters; const nameFilters = parsed.nameFilters; // The text portion drives FTS/LIKE; if all the user typed was // filters (`kind:function`), we still need *some* candidate set, // so synthesise an empty-text path that returns everything matching // the filters. const text = parsed.text; const kinds = mergedKinds; const languages = mergedLanguages; // First try FTS5 with prefix matching let results = text ? this.searchNodesFTS(text, { kinds, languages, limit, offset }) // Over-fetch by 5× when running filter-only (no text). The // post-scoring path: + name: filters can be very selective, so // a smaller multiplier risks returning fewer than `limit` // results despite the DB having plenty of matches. : this.searchAllByFilters({ kinds, languages, limit: limit * 5 }); // If no FTS results, try LIKE-based substring search if (results.length === 0 && text.length >= 2) { results = this.searchNodesLike(text, { kinds, languages, limit, offset }); } // Final fuzzy fallback: scan all known names and keep those within // a tight Levenshtein distance. Only fires when both FTS and LIKE // returned nothing AND there's a text portion long enough to be // worth fuzzing (1-char queries would match too much). if (results.length === 0 && text.length >= 3) { results = this.searchNodesFuzzy(text, { kinds, languages, limit }); } // Supplement: ensure exact name matches are always candidates. // BM25 can bury short exact-match names (e.g. "getBean") under hundreds of // compound names (e.g. "getBeanDescriptor") in large codebases, // pushing them past the FTS fetch limit before post-hoc scoring can help. // Use the max BM25 score as the base so the nameMatchBonus (exact=30 vs // prefix=20) actually differentiates them after rescoring. if (results.length > 0 && query) { const existingIds = new Set(results.map(r => r.node.id)); const maxFtsScore = Math.max(...results.map(r => r.score)); const terms = query.split(/\s+/).filter(t => t.length >= 2); for (const term of terms) { let sql = 'SELECT * FROM nodes WHERE name = ? COLLATE NOCASE'; const params: (string | number)[] = [term]; if (kinds && kinds.length > 0) { sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); } if (languages && languages.length > 0) { sql += ` AND language IN (${languages.map(() => '?').join(',')})`; params.push(...languages); } sql += ' LIMIT 20'; const rows = this.db.prepare(sql).all(...params) as NodeRow[]; for (const row of rows) { if (!existingIds.has(row.id)) { results.push({ node: rowToNode(row), score: maxFtsScore }); existingIds.add(row.id); } } } } // Apply multi-signal scoring if (results.length > 0 && (text || query)) { const scoringQuery = text || query; results = results.map(r => ({ ...r, score: r.score + kindBonus(r.node.kind) + scorePathRelevance(r.node.filePath, scoringQuery, this.projectNameTokens) + nameMatchBonus(r.node.name, scoringQuery), })); results.sort((a, b) => b.score - a.score); // Trim to requested limit after rescoring if (results.length > limit) { results = results.slice(0, limit); } } // Apply path: + name: filters AFTER scoring. Scoring already uses // path/name as a soft signal; the explicit filters here are a hard // gate. Done last so the FTS limit fetched plenty of candidates to // narrow from. if (pathFilters.length > 0) { const lowered = pathFilters.map((p) => p.toLowerCase()); results = results.filter((r) => { const fp = r.node.filePath.toLowerCase(); return lowered.some((p) => fp.includes(p)); }); } if (nameFilters.length > 0) { const lowered = nameFilters.map((n) => n.toLowerCase()); results = results.filter((r) => { const nm = r.node.name.toLowerCase(); return lowered.some((n) => nm.includes(n)); }); } return results; } /** * Match-everything path used when the user supplied only field * filters (`kind:function lang:typescript`) with no text. Returns * candidates ordered by name; the caller's filter pass narrows to * what was asked for. */ private searchAllByFilters(options: { kinds?: NodeKind[]; languages?: Language[]; limit: number; }): SearchResult[] { const { kinds, languages, limit } = options; let sql = 'SELECT * FROM nodes WHERE 1=1'; const params: (string | number)[] = []; if (kinds && kinds.length > 0) { sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); } if (languages && languages.length > 0) { sql += ` AND language IN (${languages.map(() => '?').join(',')})`; params.push(...languages); } sql += ' ORDER BY name LIMIT ?'; params.push(limit); const rows = this.db.prepare(sql).all(...params) as NodeRow[]; return rows.map((row) => ({ node: rowToNode(row), score: 1 })); } /** * Fuzzy fallback: when zero FTS/LIKE hits, try an edit-distance * sweep over the distinct symbol-name set. Caps `maxDist` at 2 so * `getUssr` finds `getUser` but `process` doesn't match `prosody`. * Bounded edit distance keeps each comparison cheap; the per-query * scan is O(distinct-name-count) which is far smaller than total * node count on any real codebase. */ private searchNodesFuzzy( text: string, options: { kinds?: NodeKind[]; languages?: Language[]; limit: number } ): SearchResult[] { const { kinds, languages, limit } = options; const lowered = text.toLowerCase(); const maxDist = lowered.length <= 4 ? 1 : 2; // Pull the distinct name list once. The set is cached on QueryBuilder // by getAllNodeNames(); even on a 200k-node project the distinct // name set is typically O(10k) because most names repeat. The // candidate-cap below bounds memory regardless. const allNames = this.getAllNodeNames(); const candidates: Array<{ name: string; dist: number }> = []; for (const name of allNames) { const dist = boundedEditDistance(name.toLowerCase(), lowered, maxDist); if (dist <= maxDist) candidates.push({ name, dist }); } candidates.sort((a, b) => a.dist - b.dist); // Cap the per-name follow-up queries. Each survivor triggers a // separate `SELECT * FROM nodes WHERE name = ?`; without this cap // a project with many similar names (`getUser1`, `getUser2`...) // could fan out far beyond `limit` queries before the inner-loop // limit kicks in. const FUZZY_FOLLOWUP_CAP = Math.max(limit * 2, 50); const cappedCandidates = candidates.slice(0, FUZZY_FOLLOWUP_CAP); const results: SearchResult[] = []; const seen = new Set(); for (const c of cappedCandidates) { if (results.length >= limit) break; let sql = 'SELECT * FROM nodes WHERE name = ?'; const params: (string | number)[] = [c.name]; if (kinds && kinds.length > 0) { sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); } if (languages && languages.length > 0) { sql += ` AND language IN (${languages.map(() => '?').join(',')})`; params.push(...languages); } sql += ' LIMIT 5'; const rows = this.db.prepare(sql).all(...params) as NodeRow[]; for (const row of rows) { if (seen.has(row.id)) continue; seen.add(row.id); // Lower the score for each edit step away from the query so // exact-match fallbacks (dist 0) outrank dist-2 typos. results.push({ node: rowToNode(row), score: 1 / (1 + c.dist) }); if (results.length >= limit) break; } } return results; } /** * FTS5 search with prefix matching */ private searchNodesFTS(query: string, options: SearchOptions): SearchResult[] { const { kinds, languages, limit = 100, offset = 0 } = options; // Add prefix wildcard for better matching (e.g., "auth" matches "AuthService", "authenticate") // Escape special FTS5 characters and add prefix wildcard. // // `::` is a qualifier separator in Rust/C++/Ruby, not a token char, // so treat it as whitespace before the strip step. Otherwise queries // like `stage_apply::run` collapse to `stage_applyrun` (the colons // are stripped without splitting) and find nothing. See #173. const ftsQuery = query .replace(/::/g, ' ') // Rust/C++/Ruby qualifier separator .replace(/['"*():^]/g, '') // Remove FTS5 special chars .split(/\s+/) .filter(term => term.length > 0) // Strip FTS5 boolean operators to prevent query manipulation .filter(term => !/^(AND|OR|NOT|NEAR)$/i.test(term)) .map(term => `"${term}"*`) // Prefix match each term .join(' OR '); if (!ftsQuery) { return []; } // BM25 column weights: id=0, name=20, qualified_name=5, docstring=1, signature=2 // Heavy name weight ensures exact/prefix name matches rank above incidental // mentions in long docstrings or qualified names of nested symbols. // Fetch 5x requested limit so post-hoc rescoring (kindBonus, pathRelevance, // nameMatchBonus) can promote results that BM25 alone undervalues. const ftsLimit = Math.max(limit * 5, 100); let sql = ` SELECT nodes.*, bm25(nodes_fts, 0, 20, 5, 1, 2) as score FROM nodes_fts JOIN nodes ON nodes_fts.id = nodes.id WHERE nodes_fts MATCH ? `; const params: (string | number)[] = [ftsQuery]; if (kinds && kinds.length > 0) { sql += ` AND nodes.kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); } if (languages && languages.length > 0) { sql += ` AND nodes.language IN (${languages.map(() => '?').join(',')})`; params.push(...languages); } sql += ' ORDER BY score LIMIT ? OFFSET ?'; params.push(ftsLimit, offset); try { const rows = this.db.prepare(sql).all(...params) as (NodeRow & { score: number })[]; return rows.map((row) => ({ node: rowToNode(row), score: Math.abs(row.score), // bm25 returns negative scores })); } catch { // FTS query failed, return empty return []; } } /** * LIKE-based substring search for cases where FTS doesn't match * Useful for camelCase matching (e.g., "signIn" finds "signInWithGoogle") */ private searchNodesLike(query: string, options: SearchOptions): SearchResult[] { const { kinds, languages, limit = 100, offset = 0 } = options; let sql = ` SELECT nodes.*, CASE WHEN name = ? THEN 1.0 WHEN name LIKE ? THEN 0.9 WHEN name LIKE ? THEN 0.8 WHEN qualified_name LIKE ? THEN 0.7 ELSE 0.5 END as score FROM nodes WHERE ( name LIKE ? OR qualified_name LIKE ? OR name LIKE ? ) `; // Pattern variants for better matching const exactMatch = query; const startsWith = `${query}%`; const contains = `%${query}%`; const params: (string | number)[] = [ exactMatch, // Exact match score startsWith, // Starts with score contains, // Contains score contains, // Qualified name score contains, // WHERE: name contains contains, // WHERE: qualified_name contains startsWith, // WHERE: name starts with ]; if (kinds && kinds.length > 0) { sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); } if (languages && languages.length > 0) { sql += ` AND language IN (${languages.map(() => '?').join(',')})`; params.push(...languages); } sql += ' ORDER BY score DESC, length(name) ASC LIMIT ? OFFSET ?'; params.push(limit, offset); const rows = this.db.prepare(sql).all(...params) as (NodeRow & { score: number })[]; return rows.map((row) => ({ node: rowToNode(row), score: row.score, })); } /** * Find nodes by exact name match * * Used for hybrid search - looks up symbols by exact name or case-insensitive match. * Returns high-confidence matches for known symbol names extracted from query. * * @param names - Array of symbol names to look up * @param options - Search options (kinds, languages, limit) * @returns SearchResult array with exact matches scored at 1.0 */ findNodesByExactName(names: string[], options: SearchOptions = {}): SearchResult[] { if (names.length === 0) return []; const { kinds, languages, limit = 50 } = options; // Two-pass approach to handle common names (e.g., "run" has 40+ matches): // Pass 1: Find which files contain distinctive (rare) symbols from the query. // Pass 2: Query each name, boosting results that co-locate with distinctive symbols. // Pass 1: Find files containing each queried name, identify distinctive names const nameToFiles = new Map>(); for (const name of names) { let sql = 'SELECT DISTINCT file_path FROM nodes WHERE name COLLATE NOCASE = ?'; const params: (string | number)[] = [name]; if (kinds && kinds.length > 0) { sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); } sql += ' LIMIT 100'; const rows = this.db.prepare(sql).all(...params) as { file_path: string }[]; nameToFiles.set(name.toLowerCase(), new Set(rows.map(r => r.file_path))); } // Distinctive names are those with fewer than 10 file matches (e.g., "scrapeLoop" = 1 file) const distinctiveFiles = new Set(); for (const [, files] of nameToFiles) { if (files.size > 0 && files.size < 10) { for (const f of files) distinctiveFiles.add(f); } } // Pass 2: Query each name with per-name limit, scoring by co-location const perNameLimit = Math.max(8, Math.ceil(limit / names.length)); const allResults: SearchResult[] = []; const seenIds = new Set(); for (const name of names) { let sql = ` SELECT nodes.*, 1.0 as score FROM nodes WHERE name COLLATE NOCASE = ? `; const params: (string | number)[] = [name]; if (kinds && kinds.length > 0) { sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); } if (languages && languages.length > 0) { sql += ` AND language IN (${languages.map(() => '?').join(',')})`; params.push(...languages); } // Fetch enough to find co-located results among common names sql += ' LIMIT ?'; params.push(Math.max(perNameLimit * 3, 50)); const rows = this.db.prepare(sql).all(...params) as (NodeRow & { score: number })[]; const nameResults: SearchResult[] = []; for (const row of rows) { const node = rowToNode(row); if (seenIds.has(node.id)) continue; // Boost results in files that also contain distinctive symbols const coLocationBoost = distinctiveFiles.has(node.filePath) ? 20 : 0; nameResults.push({ node, score: row.score + coLocationBoost }); } // Sort by score (co-located first), take per-name limit nameResults.sort((a, b) => b.score - a.score); for (const r of nameResults.slice(0, perNameLimit)) { seenIds.add(r.node.id); allResults.push(r); } } // Sort all results by score so co-located results bubble up allResults.sort((a, b) => b.score - a.score); return allResults.slice(0, limit); } /** * Find nodes whose name contains a substring (LIKE-based). * Useful for CamelCase-part matching where FTS fails because * e.g. "TransportSearchAction" is one FTS token, not matchable by "Search"*. * * Results are ordered by name length (shorter = more likely to be the core type). */ findNodesByNameSubstring( substring: string, options: SearchOptions & { excludePrefix?: boolean } = {} ): SearchResult[] { const { kinds, languages, limit = 30, excludePrefix } = options; let sql = ` SELECT nodes.*, 1.0 as score FROM nodes WHERE name LIKE ? `; const params: (string | number)[] = [`%${substring}%`]; // Exclude prefix matches (handled by FTS-based prefix search in Step 2b) if (excludePrefix) { sql += ` AND name NOT LIKE ?`; params.push(`${substring}%`); } if (kinds && kinds.length > 0) { sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); } if (languages && languages.length > 0) { sql += ` AND language IN (${languages.map(() => '?').join(',')})`; params.push(...languages); } sql += ' ORDER BY length(name) ASC LIMIT ?'; params.push(limit); const rows = this.db.prepare(sql).all(...params) as (NodeRow & { score: number })[]; return rows.map((row) => ({ node: rowToNode(row), score: row.score, })); } // =========================================================================== // Edge Operations // =========================================================================== /** * Insert a new edge */ insertEdge(edge: Edge): void { if (!this.stmts.insertEdge) { this.stmts.insertEdge = this.db.prepare(` INSERT OR IGNORE INTO edges (source, target, kind, metadata, line, col, provenance) VALUES (@source, @target, @kind, @metadata, @line, @col, @provenance) `); } this.stmts.insertEdge.run({ source: edge.source, target: edge.target, kind: edge.kind, metadata: edge.metadata ? JSON.stringify(edge.metadata) : null, line: edge.line ?? null, col: edge.column ?? null, provenance: edge.provenance ?? null, }); } /** * Insert multiple edges in a transaction */ insertEdges(edges: Edge[]): void { if (edges.length === 0) return; this.db.transaction(() => { const endpointIds = new Set(); for (const edge of edges) { endpointIds.add(edge.source); endpointIds.add(edge.target); } const existingNodeIds = this.getExistingNodeIds([...endpointIds]); const rows: unknown[][] = []; for (const edge of edges) { if (!existingNodeIds.has(edge.source) || !existingNodeIds.has(edge.target)) { continue; } rows.push([ edge.source, edge.target, edge.kind, edge.metadata ? JSON.stringify(edge.metadata) : null, edge.line ?? null, edge.column ?? null, edge.provenance ?? null, ]); } this.runBatched( 'insertEdges', 'INSERT OR IGNORE INTO edges (source, target, kind, metadata, line, col, provenance) VALUES ', '(?,?,?,?,?,?,?)', rows ); })(); } /** * Delete all edges from a source node */ deleteEdgesBySource(sourceId: string): void { if (!this.stmts.deleteEdgesBySource) { this.stmts.deleteEdgesBySource = this.db.prepare('DELETE FROM edges WHERE source = ?'); } this.stmts.deleteEdgesBySource.run(sourceId); } /** * Get outgoing edges from a node */ getOutgoingEdges(sourceId: string, kinds?: EdgeKind[], provenance?: string): Edge[] { if ((kinds && kinds.length > 0) || provenance) { let sql = 'SELECT * FROM edges WHERE source = ?'; const params: (string | number)[] = [sourceId]; if (kinds && kinds.length > 0) { sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); } if (provenance) { sql += ' AND provenance = ?'; params.push(provenance); } const rows = this.db.prepare(sql).all(...params) as EdgeRow[]; return rows.map(rowToEdge); } if (!this.stmts.getEdgesBySource) { this.stmts.getEdgesBySource = this.db.prepare('SELECT * FROM edges WHERE source = ?'); } const rows = this.stmts.getEdgesBySource.all(sourceId) as EdgeRow[]; return rows.map(rowToEdge); } /** * Get incoming edges to a node */ getIncomingEdges(targetId: string, kinds?: EdgeKind[]): Edge[] { if (kinds && kinds.length > 0) { const sql = `SELECT * FROM edges WHERE target = ? AND kind IN (${kinds.map(() => '?').join(',')})`; const rows = this.db.prepare(sql).all(targetId, ...kinds) as EdgeRow[]; return rows.map(rowToEdge); } if (!this.stmts.getEdgesByTarget) { this.stmts.getEdgesByTarget = this.db.prepare('SELECT * FROM edges WHERE target = ?'); } const rows = this.stmts.getEdgesByTarget.all(targetId) as EdgeRow[]; return rows.map(rowToEdge); } /** * Find all edges where both source and target are in the given node set. * Useful for recovering inter-node connectivity after BFS. */ findEdgesBetweenNodes(nodeIds: string[], kinds?: EdgeKind[]): Edge[] { if (nodeIds.length === 0) return []; const idsJson = JSON.stringify(nodeIds); let sql = `SELECT * FROM edges WHERE source IN (SELECT value FROM json_each(?)) AND target IN (SELECT value FROM json_each(?))`; const params: string[] = [idsJson, idsJson]; if (kinds && kinds.length > 0) { sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); } const rows = this.db.prepare(sql).all(...params) as EdgeRow[]; return rows.map(rowToEdge); } /** * Distinct file paths that DEPEND ON `filePath`: every file containing a * symbol with a cross-file edge (any kind except `contains`) into a symbol * of this file. This is the file-level projection of the symbol dependency * graph and the basis for blast-radius / `affected` test selection. * * It deliberately does NOT restrict to `imports` edges. In this graph an * `imports` edge connects a file to its own local import declarations * (it is always same-file), so an imports-only lookup returns zero * cross-file dependents for every file. The real cross-file dependency * signal is the resolved call/reference graph — calls, references, * instantiates, extends, implements, overrides, type_of, returns, * decorates — exactly what {@link GraphTraverser.getImpactRadius} traverses. * `contains` is excluded: a parent containing a symbol does not *depend* on * it. One indexed query (idx_nodes_file_path + idx_edges_target_kind). */ getDependentFilePaths(filePath: string): string[] { const sql = `SELECT DISTINCT src.file_path AS fp FROM edges e JOIN nodes tgt ON tgt.id = e.target JOIN nodes src ON src.id = e.source WHERE tgt.file_path = ? AND e.kind != 'contains' AND src.file_path != ?`; const rows = this.db.prepare(sql).all(filePath, filePath) as Array<{ fp: string }>; return rows.map((r) => r.fp); } /** * Distinct file paths that `filePath` DEPENDS ON — the inverse of * {@link getDependentFilePaths}: every file containing a symbol that a * symbol of this file has a cross-file edge into. Same edge-kind rules * (all kinds except `contains`); same reason imports-only is insufficient. */ getDependencyFilePaths(filePath: string): string[] { const sql = `SELECT DISTINCT tgt.file_path AS fp FROM edges e JOIN nodes src ON src.id = e.source JOIN nodes tgt ON tgt.id = e.target WHERE src.file_path = ? AND e.kind != 'contains' AND tgt.file_path != ?`; const rows = this.db.prepare(sql).all(filePath, filePath) as Array<{ fp: string }>; return rows.map((r) => r.fp); } /** * Cross-file edges whose TARGET is a node in `filePath` and whose SOURCE is a * node in a *different* file, paired with the target node's (name, kind) so a * caller can re-resolve the edge to the re-indexed target's new ID (node IDs * are `sha256(filePath:kind:name:line)`, so any line shift in the callee file * changes target IDs and a naive re-insert by old ID silently drops them). * Used by `storeExtractionResult` to preserve incoming edges across a file * re-index (issue #899). Same edge-kind rules as * {@link getDependentFilePaths}: all kinds except `contains`. */ getCrossFileIncomingEdgesWithTarget( filePath: string ): Array { const sql = `SELECT e.*, tgt.name AS target_name, tgt.kind AS target_kind, src.file_path AS source_file_path, src.language AS source_language FROM edges e JOIN nodes tgt ON tgt.id = e.target JOIN nodes src ON src.id = e.source WHERE tgt.file_path = ? AND e.kind != 'contains' AND src.file_path != ?`; const rows = this.db.prepare(sql).all(filePath, filePath) as Array< EdgeRow & { target_name: string; target_kind: NodeKind; source_file_path: string; source_language: Language } >; return rows.map(row => ({ ...rowToEdge(row), targetName: row.target_name, targetKind: row.target_kind, sourceFilePath: row.source_file_path, sourceLanguage: row.source_language, })); } // =========================================================================== // File Operations // =========================================================================== /** * Insert or update a file record */ upsertFile(file: FileRecord): void { if (!this.stmts.upsertFile) { this.stmts.upsertFile = this.db.prepare(` INSERT INTO files (path, content_hash, language, size, modified_at, indexed_at, node_count, errors, generated) VALUES (@path, @contentHash, @language, @size, @modifiedAt, @indexedAt, @nodeCount, @errors, @generated) ON CONFLICT(path) DO UPDATE SET content_hash = @contentHash, language = @language, size = @size, modified_at = @modifiedAt, indexed_at = @indexedAt, node_count = @nodeCount, errors = @errors, generated = @generated `); } this.stmts.upsertFile.run({ path: file.path, contentHash: file.contentHash, language: file.language, size: file.size, modifiedAt: file.modifiedAt, indexedAt: file.indexedAt, nodeCount: file.nodeCount, errors: file.errors ? JSON.stringify(file.errors) : null, // The upsert always REWRITES the flag: a file that loses its banner in an // edit must lose the flag on the next sync, not keep a stale 1. generated: file.generated ? 1 : 0, }); } /** * Which of `filePaths` the index flagged as tool-generated (schema v9+). * * Bounded-lookup by design: every consumer already holds a short candidate * list (a ranked file group, an FTS result page, a LIMIT-20 aggregate), so * this stays a partial-index probe over a handful of paths — no whole-repo * set to materialize, and no cache to invalidate, which means a ranking call * can never serve a verdict the last sync already replaced. * * Returns ONLY the content/index signal; callers union it with * {@link isGeneratedFile} so pre-v9 databases (column present, all zeros * until a re-index) keep the path-only behavior rather than regressing. */ getGeneratedPathsAmong(filePaths: Iterable): Set { const unique = [...new Set(filePaths)]; const found = new Set(); if (unique.length === 0) return found; for (let i = 0; i < unique.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = unique.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); const rows = this.db .prepare(`SELECT path FROM files WHERE generated = 1 AND path IN (${placeholders})`) .all(...chunk) as Array<{ path: string }>; for (const row of rows) found.add(row.path); } return found; } /** * A reusable `(path) => boolean` over a bounded candidate list, unioning the * indexed flag with the path convention. This is the shape every ranking * comparator wants: one query up front, then O(1) per comparison. */ generatedPredicateFor(filePaths: Iterable): (filePath: string) => boolean { const flagged = this.getGeneratedPathsAmong(filePaths); return (filePath: string) => flagged.has(filePath) || isGeneratedFile(filePath); } /** * Which of `filePaths` are AMBIENT DECLARATION files — they declare nothing * but types, and nothing in the index depends on them (CG-28). A hand-written * ambient `.d.ts` of global shims, a vendored typings file, module * augmentation: reachable only by name, structurally attached to nothing. * * Structural, not extension-based, so a hand-written `types.ts` and a `.d.ts` * are judged by the same rule and a `.d.ts` that does declare a class or a * const is (correctly) not caught. Four conditions, all required: * * 1. it declares at least one symbol — an empty or unparsed file is not a * declaration file, it is a file we know nothing about; * 2. EVERY declared symbol is a type-level kind (interface / type alias / * enum / namespace). The narrowness is deliberate and measured: a rule * of "no callables" alone flags 1–18% of a repo, including Kotlin sealed * classes, Rust `mod.rs` re-exports and django's locale constant tables — * real source that must not be demoted. This rule flags 0–4%; * 3. no symbol in it originates a `calls`/`instantiates` edge — the direct * evidence that nothing here has a body; * 4. NOTHING ELSE IN THE INDEX points at it. This is the condition that * separates an ambient shim from a working type module, and it is why * the flag is narrow enough to be safe: `displacement-ts`'s pipeline * `types.ts` passes 1–3 identically but carries 13 inbound imports and * 21 references, so the files that answer a query about the pipeline are * typed BY it — it is part of that answer's structure. An ambient * `declare global` shim has zero. Deliberately index-wide rather than * restricted to the candidate list: the file that imports it is usually * not itself a candidate. * * Bounded-lookup like {@link getGeneratedPathsAmong}: callers hold a ranked * candidate list, so this is a partial-index probe over a handful of paths. */ getAmbientDeclarationPathsAmong(filePaths: Iterable): Set { const unique = [...new Set(filePaths)]; const found = new Set(); if (unique.length === 0) return found; for (let i = 0; i < unique.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = unique.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); // `file`/`import`/`export`/`parameter` are structural bookkeeping, not // things the file declares, so they neither qualify nor disqualify. const rows = this.db .prepare(` SELECT file_path, SUM(CASE WHEN kind NOT IN ('file','import','export','parameter') THEN 1 ELSE 0 END) AS declared, SUM(CASE WHEN kind IN ('interface','type_alias','enum','enum_member','namespace') THEN 1 ELSE 0 END) AS typeDeclared FROM nodes WHERE file_path IN (${placeholders}) GROUP BY file_path `) .all(...chunk) as Array<{ file_path: string; declared: number; typeDeclared: number }>; let candidates = rows .filter((r) => r.declared > 0 && r.declared === r.typeDeclared) .map((r) => r.file_path); if (candidates.length === 0) continue; const disqualify = (sql: string): void => { if (candidates.length === 0) return; const hit = new Set( (this.db .prepare(sql.replace('$IN$', candidates.map(() => '?').join(','))) .all(...candidates) as Array<{ file_path: string }>).map((r) => r.file_path), ); candidates = candidates.filter((p) => !hit.has(p)); }; // (3) originates behaviour disqualify(` SELECT DISTINCT n.file_path AS file_path FROM edges e JOIN nodes n ON n.id = e.source WHERE e.kind IN ('calls','instantiates') AND n.file_path IN ($IN$) `); // (4) something outside the file depends on it disqualify(` SELECT DISTINCT t.file_path AS file_path FROM edges e JOIN nodes t ON t.id = e.target JOIN nodes s ON s.id = e.source WHERE t.file_path IN ($IN$) AND s.file_path <> t.file_path `); for (const path of candidates) found.add(path); } return found; } /** * A reusable `(path) => boolean` ambient-declaration test over a bounded * candidate list — the shape a ranking comparator wants: one query up front, * O(1) per comparison. */ ambientDeclarationPredicateFor(filePaths: Iterable): (filePath: string) => boolean { const flagged = this.getAmbientDeclarationPathsAmong(filePaths); return (filePath: string) => flagged.has(filePath); } /** How many indexed files carry the generated flag. Surfaced by `status`. */ countGeneratedFiles(): number { const row = this.db .prepare('SELECT COUNT(*) AS n FROM files WHERE generated = 1') .get() as { n: number } | undefined; return row?.n ?? 0; } /** * Delete a file record and its nodes */ deleteFile(filePath: string): void { this.db.transaction(() => { this.deleteNodesByFile(filePath); if (!this.stmts.deleteFile) { this.stmts.deleteFile = this.db.prepare('DELETE FROM files WHERE path = ?'); } this.stmts.deleteFile.run(filePath); })(); } /** * Get a file record by path */ getFileByPath(filePath: string): FileRecord | null { if (!this.stmts.getFileByPath) { this.stmts.getFileByPath = this.db.prepare('SELECT * FROM files WHERE path = ?'); } const row = this.stmts.getFileByPath.get(filePath) as FileRow | undefined; return row ? rowToFileRecord(row) : null; } /** * Get all tracked files */ getAllFiles(): FileRecord[] { if (!this.stmts.getAllFiles) { this.stmts.getAllFiles = this.db.prepare('SELECT * FROM files ORDER BY path'); } const rows = this.stmts.getAllFiles.all() as FileRow[]; return rows.map(rowToFileRecord); } /** * Most recent index timestamp (ms since epoch) across all tracked files, or * null when nothing is indexed yet. One indexed aggregate, no per-row scan. (#329) */ getLastIndexedAt(): number | null { const row = this.db .prepare('SELECT MAX(indexed_at) AS last FROM files') .get() as { last: number | null } | undefined; return row?.last ?? null; } /** * Get files that need re-indexing (hash changed) */ getStaleFiles(currentHashes: Map): FileRecord[] { const files = this.getAllFiles(); return files.filter((f) => { const currentHash = currentHashes.get(f.path); return currentHash && currentHash !== f.contentHash; }); } // =========================================================================== // Unresolved References // =========================================================================== /** * Insert an unresolved reference */ insertUnresolvedRef(ref: UnresolvedReference): void { if (!this.stmts.insertUnresolved) { this.stmts.insertUnresolved = this.db.prepare(` INSERT INTO unresolved_refs (from_node_id, reference_name, reference_kind, line, col, candidates, file_path, language) VALUES (@fromNodeId, @referenceName, @referenceKind, @line, @col, @candidates, @filePath, @language) `); } this.stmts.insertUnresolved.run({ fromNodeId: ref.fromNodeId, referenceName: ref.referenceName, referenceKind: ref.referenceKind, line: ref.line, col: ref.column, candidates: ref.candidates ? JSON.stringify(ref.candidates) : null, filePath: ref.filePath ?? '', language: ref.language ?? 'unknown', }); } /** * Insert multiple unresolved references in a transaction */ insertUnresolvedRefsBatch(refs: UnresolvedReference[]): void { if (refs.length === 0) return; const insert = this.db.transaction(() => { const rows: unknown[][] = []; for (const ref of refs) { rows.push([ ref.fromNodeId, ref.referenceName, ref.referenceKind, ref.line, ref.column, ref.candidates ? JSON.stringify(ref.candidates) : null, ref.filePath ?? '', ref.language ?? 'unknown', ]); } this.runBatched( 'insertUnresolvedRefs', 'INSERT INTO unresolved_refs (from_node_id, reference_name, reference_kind, line, col, candidates, file_path, language) VALUES ', '(?,?,?,?,?,?,?,?)', rows ); }); insert(); } /** * Delete unresolved references from a node */ deleteUnresolvedByNode(nodeId: string): void { if (!this.stmts.deleteUnresolvedByNode) { this.stmts.deleteUnresolvedByNode = this.db.prepare( 'DELETE FROM unresolved_refs WHERE from_node_id = ?' ); } this.stmts.deleteUnresolvedByNode.run(nodeId); } /** * Get unresolved references by name (for resolution) */ getUnresolvedByName(name: string): UnresolvedReference[] { if (!this.stmts.getUnresolvedByName) { this.stmts.getUnresolvedByName = this.db.prepare( 'SELECT * FROM unresolved_refs WHERE reference_name = ?' ); } const rows = this.stmts.getUnresolvedByName.all(name) as UnresolvedRefRow[]; return rows.map((row) => ({ fromNodeId: row.from_node_id, referenceName: row.reference_name, referenceKind: row.reference_kind as EdgeKind, line: row.line, column: row.col, candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, filePath: row.file_path, language: row.language as Language, rowId: row.id, })); } /** * Get all unresolved references */ getUnresolvedReferences(): UnresolvedReference[] { const rows = this.db.prepare('SELECT * FROM unresolved_refs').all() as UnresolvedRefRow[]; return rows.map((row) => ({ fromNodeId: row.from_node_id, referenceName: row.reference_name, referenceKind: row.reference_kind as EdgeKind, line: row.line, column: row.col, candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, filePath: row.file_path, language: row.language as Language, rowId: row.id, })); } /** * Get the count of PENDING (never-attempted) references without loading * them into memory. Rows marked status='failed' — attempted by a completed * pass, no match — are excluded: they are not outstanding work, only retry * candidates for the #1240 sweep, so they must not trip the #1187 orphan * sweep or the `status` pending-refs warning. */ getUnresolvedReferencesCount(): number { if (!this.stmts.getUnresolvedCount) { this.stmts.getUnresolvedCount = this.db.prepare( "SELECT COUNT(*) as count FROM unresolved_refs WHERE status = 'pending'" ); } const row = this.stmts.getUnresolvedCount.get() as { count: number }; return row.count; } /** * Get a batch of PENDING unresolved references using LIMIT/OFFSET * pagination. Used to process references in bounded memory chunks; failed * rows are excluded so the batched drain loop terminates once every row * has been attempted. */ getUnresolvedReferencesBatch(offset: number, limit: number): UnresolvedReference[] { if (!this.stmts.getUnresolvedBatch) { // ORDER BY rowid is load-bearing for the pipelined resolution loop: it // prefetches batch k+1 at OFFSET batch_k.length while batch k's rows are // still pending, which is only exact under a stable enumeration. (A plain // scan and the status index both return rowid order anyway — this pins // it.) this.stmts.getUnresolvedBatch = this.db.prepare( "SELECT * FROM unresolved_refs WHERE status = 'pending' ORDER BY rowid LIMIT ? OFFSET ?" ); } const rows = this.stmts.getUnresolvedBatch.all(limit, offset) as UnresolvedRefRow[]; return rows.map((row) => ({ fromNodeId: row.from_node_id, referenceName: row.reference_name, referenceKind: row.reference_kind as EdgeKind, line: row.line, column: row.col, candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, filePath: row.file_path, language: row.language as Language, rowId: row.id, })); } /** * Keyset variant of {@link getUnresolvedReferencesBatch} for the batched * resolution loop: seek past the last-seen row id instead of OFFSET-walking. * OFFSET reads re-scan the accumulated failed-row prefix on every batch — * O(failed rows) per read, measured at 54.6s of the kernel-scale batch loop * (§7a.2) — while the seek is O(batch) forever. `id` is the rowid alias, so * the enumeration order is identical to the OFFSET reader's. */ getUnresolvedReferencesBatchAfter(afterRowId: number, limit: number): UnresolvedReference[] { if (!this.stmts.getUnresolvedBatchAfter) { this.stmts.getUnresolvedBatchAfter = this.db.prepare( "SELECT * FROM unresolved_refs WHERE status = 'pending' AND id > ? ORDER BY id LIMIT ?" ); } const rows = this.stmts.getUnresolvedBatchAfter.all(afterRowId, limit) as UnresolvedRefRow[]; return rows.map((row) => ({ fromNodeId: row.from_node_id, referenceName: row.reference_name, referenceKind: row.reference_kind as EdgeKind, line: row.line, column: row.col, candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, filePath: row.file_path, language: row.language as Language, rowId: row.id, })); } /** * Get all tracked file paths (lightweight — no full FileRecord objects) */ getAllFilePaths(): string[] { if (!this.stmts.getAllFilePaths) { this.stmts.getAllFilePaths = this.db.prepare('SELECT path FROM files ORDER BY path'); } const rows = this.stmts.getAllFilePaths.all() as Array<{ path: string }>; return rows.map((r) => r.path); } /** * Get all distinct node names (lightweight — just name strings for pre-filtering) */ getAllNodeNames(): string[] { if (!this.stmts.getAllNodeNames) { this.stmts.getAllNodeNames = this.db.prepare('SELECT DISTINCT name FROM nodes'); } const rows = this.stmts.getAllNodeNames.all() as Array<{ name: string }>; return rows.map((r) => r.name); } /** * Stream the distinct node names one row at a time — the incremental * counterpart to {@link getAllNodeNames} for callers that need to yield * to the event loop mid-scan (resolver cache warm-up on multi-million-node * indexes). Fresh statement per call: the iterator holds an open cursor. */ *iterateNodeNames(): IterableIterator { const stmt = this.db.prepare('SELECT DISTINCT name FROM nodes'); for (const row of stmt.iterate()) { yield (row as { name: string }).name; } } /** * Get unresolved references scoped to specific file paths. * Uses the idx_unresolved_file_path index for efficient lookup. */ getUnresolvedReferencesByFiles(filePaths: string[]): UnresolvedReference[] { if (filePaths.length === 0) return []; // Chunk under SQLite's parameter limit: the first sync of a very large repo // passes every changed file here, which an unbounded `IN (...)` would bind // as one parameter each — exceeding MAX_VARIABLE_NUMBER and aborting with // "too many SQL variables". (#540) const rows: UnresolvedRefRow[] = []; for (let i = 0; i < filePaths.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = filePaths.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); const chunkRows = this.db .prepare(`SELECT * FROM unresolved_refs WHERE status = 'pending' AND file_path IN (${placeholders})`) .all(...chunk) as UnresolvedRefRow[]; // Append with a loop, never a spread: the INPUT chunk is bounded, but // the RESULT rows per chunk are not — a dense recovery sync (e.g. the // #1541 self-heal re-indexing hundreds of files) returns more rows than // V8 allows as arguments, and `push(...chunkRows)` dies with "Maximum // call stack size exceeded", aborting resolution mid-sync (#1558). for (const row of chunkRows) rows.push(row); } return rows.map((row) => ({ fromNodeId: row.from_node_id, referenceName: row.reference_name, referenceKind: row.reference_kind as EdgeKind, line: row.line, column: row.col, candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, filePath: row.file_path, language: row.language as Language, rowId: row.id, })); } /** * Delete all unresolved references (after resolution) */ clearUnresolvedReferences(): void { this.db.exec('DELETE FROM unresolved_refs'); } /** * Delete resolved references by their IDs */ deleteResolvedReferences(fromNodeIds: string[]): void { if (fromNodeIds.length === 0) return; // Chunk under SQLite's parameter limit, matching every other IN-list in // this file. The internal resolution path uses deleteSpecificResolvedReferences // instead, but QueryBuilder is part of the public API, so a library consumer // passing more ids than SQLITE_MAX_VARIABLE_NUMBER (32766 on the bundled // node:sqlite) would otherwise hit "too many SQL variables". (#540, #1001) for (let i = 0; i < fromNodeIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = fromNodeIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); this.db.prepare(`DELETE FROM unresolved_refs WHERE from_node_id IN (${placeholders})`).run(...chunk); } } /** * Delete specific resolved references by (fromNodeId, referenceName, referenceKind) tuples. * More precise than deleteResolvedReferences — only removes refs that were actually resolved. */ deleteSpecificResolvedReferences(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): number { if (refs.length === 0) return 0; const stmt = this.db.prepare( 'DELETE FROM unresolved_refs WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?' ); // Returns rows actually removed (SQLite `changes`, summed): the batched // resolution loop's non-progress guard keys on this — zero removals from // a batch that claimed work is the direct runaway signal (§7a.2). let changed = 0; const deleteMany = this.db.transaction((items: typeof refs) => { for (const ref of items) { changed += stmt.run(ref.fromNodeId, ref.referenceName, ref.referenceKind).changes; } }); deleteMany(refs); return changed; } /** * Delete unresolved-ref rows by row id — the precise cleanup for refs a * resolution pass actually processed. The key-tuple variant above also * deletes SIBLING rows (same caller calling the same callee at other lines) * that a later batch hasn't attempted yet, so when a batch boundary split a * caller's same-named call sites, the later sites' edges were silently never * created (#1269). */ deleteReferencesByRowIds(rowIds: number[]): number { if (rowIds.length === 0) return 0; // One transaction for all chunks (each chunk was previously its own // implicit transaction = its own WAL commit — measurable on 100k+-ref // resolution persists), and the full-size chunk statement is cached so // repeat calls skip the re-prepare; only the final partial chunk (if any) // prepares ad hoc. Returns rows actually removed (summed `changes`) for // the batched loop's non-progress guard (§7a.2). let changed = 0; this.db.transaction(() => { for (let i = 0; i < rowIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = rowIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); if (chunk.length === SQLITE_PARAM_CHUNK_SIZE) { if (!this.stmts.deleteRefsByRowIdsFull) { const placeholders = new Array(SQLITE_PARAM_CHUNK_SIZE).fill('?').join(','); this.stmts.deleteRefsByRowIdsFull = this.db.prepare( `DELETE FROM unresolved_refs WHERE id IN (${placeholders})` ); } changed += this.stmts.deleteRefsByRowIdsFull.run(...chunk).changes; } else { const placeholders = chunk.map(() => '?').join(','); changed += this.db.prepare(`DELETE FROM unresolved_refs WHERE id IN (${placeholders})`).run(...chunk).changes; } } })(); return changed; } /** * Mark refs a completed resolution pass could not resolve as status='failed' * instead of deleting them (#1240). Failed rows are invisible to the pending * count/batch readers (so drain loops and the #1187 orphan sweep still * terminate) but stay queryable by name_tail so a later sync can retry them * when a changed file introduces a symbol that could satisfy them. name_tail * is (re)written here so rows inserted before the v8 migration get their * tail the first time they're attempted. */ markReferencesFailed(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): number { if (refs.length === 0) return 0; const stmt = this.db.prepare( "UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?" ); let changed = 0; const markMany = this.db.transaction((items: typeof refs) => { for (const ref of items) { changed += stmt.run(referenceNameTail(ref.referenceName), ref.fromNodeId, ref.referenceName, ref.referenceKind).changes; } }); markMany(refs); return changed; } /** * Park refs as status='failed' by row id — the precise counterpart of * markReferencesFailed, for the same reason as deleteReferencesByRowIds: * the key-tuple variant also flips same-key sibling rows in later batches * to 'failed' before they were ever attempted (#1269). Resolution outcome * can differ per call site (receiver-type inference reads the ref's line), * so a sibling must not inherit this row's failure. */ markReferencesFailedByRowIds(refs: Array<{ rowId: number; referenceName: string }>): number { if (refs.length === 0) return 0; const stmt = this.db.prepare( "UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE id = ?" ); let changed = 0; const markMany = this.db.transaction((items: typeof refs) => { for (const ref of items) { changed += stmt.run(referenceNameTail(ref.referenceName), ref.rowId).changes; } }); markMany(refs); return changed; } /** * Failed refs whose name tail matches one of the given symbol names — the * candidates a sync should retry after files carrying those names changed * (#1240). Names matching more than `perNameCeiling` failed refs are * skipped entirely: at that population a name is external/builtin noise * (`get`, `map`, …) that one new definition won't resolve — the same * rationale as resolution's AMBIGUOUS_NAME_CEILING (#999) — and retrying an * arbitrary subset would be both wasted work and incoherent coverage. */ getRetryableFailedReferences(names: string[], perNameCeiling: number = 500): UnresolvedReference[] { if (names.length === 0) return []; // Pass 1: per-tail counts, chunked under the SQLite parameter limit. const retryNames: string[] = []; for (let i = 0; i < names.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = names.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); const counts = this.db .prepare( `SELECT name_tail, COUNT(*) as count FROM unresolved_refs WHERE status = 'failed' AND name_tail IN (${placeholders}) GROUP BY name_tail` ) .all(...chunk) as Array<{ name_tail: string; count: number }>; for (const row of counts) { if (row.count <= perNameCeiling) retryNames.push(row.name_tail); } } if (retryNames.length === 0) return []; // Pass 2: load the surviving rows. const rows: UnresolvedRefRow[] = []; for (let i = 0; i < retryNames.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = retryNames.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); const chunkRows = this.db .prepare(`SELECT * FROM unresolved_refs WHERE status = 'failed' AND name_tail IN (${placeholders})`) .all(...chunk) as UnresolvedRefRow[]; // Loop, not spread — same V8 argument-limit hazard as // getUnresolvedReferencesByFiles (#1558): a large definition delta can // select an unbounded number of failed rows per chunk. for (const row of chunkRows) rows.push(row); } return rows.map((row) => ({ fromNodeId: row.from_node_id, referenceName: row.reference_name, referenceKind: row.reference_kind as EdgeKind, line: row.line, column: row.col, candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, filePath: row.file_path, language: row.language as Language, rowId: row.id, })); } /** * Resolution edges whose TARGET symbol is named one of `names` — the edges a * sync must re-resolve after `names` gained or lost a definition (CG-33). * * Resolution binds a reference to a node whose name matches the reference's * tail, and it picks among ALL same-named definitions project-wide. So adding * or removing one definition of `pct` changes the answer for every `pct(...)` * reference in the repo — including references in files this sync never * touches, whose edges nothing else revisits. Those edges' current target is, * by that same rule, a node named `pct`, which is why the target's name is a * sufficient (and index-backed, via idx_nodes_name) way to find them without * a schema change or a scan of edge metadata. * * Returns the source file/language alongside each edge so the caller can * resurrect it as its original reference. Excludes `provenance='heuristic'` * (synthesized dispatch edges are not resolution output and carry no refName * stamp to resurrect from — deleting one would be a permanent loss). * * Names matching more than `perNameCeiling` edges are skipped entirely, same * rationale and same default as {@link getRetryableFailedReferences}: at that * population the name is generic (`get`, `clear`, …), one definition changing * won't flip most of them, and rebinding an arbitrary subset is both wasted * work and incoherent coverage. */ getResolutionEdgesByTargetName( names: string[], perNameCeiling: number = 500 ): Array { if (names.length === 0) return []; // Pass 1: per-name edge counts, chunked under the SQLite parameter limit. const keep: string[] = []; for (let i = 0; i < names.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = names.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); const counts = this.db .prepare( `SELECT tgt.name AS name, COUNT(*) AS count FROM edges e JOIN nodes tgt ON tgt.id = e.target WHERE tgt.name IN (${placeholders}) AND (e.provenance IS NULL OR e.provenance != 'heuristic') GROUP BY tgt.name` ) .all(...chunk) as Array<{ name: string; count: number }>; for (const row of counts) { if (row.count <= perNameCeiling) keep.push(row.name); } } if (keep.length === 0) return []; // Pass 2: load the surviving edges with the source file context a // resurrection needs. const out: Array = []; for (let i = 0; i < keep.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = keep.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); const rows = this.db .prepare( `SELECT e.*, src.file_path AS source_file_path, src.language AS source_language FROM edges e JOIN nodes tgt ON tgt.id = e.target JOIN nodes src ON src.id = e.source WHERE tgt.name IN (${placeholders}) AND (e.provenance IS NULL OR e.provenance != 'heuristic')` ) .all(...chunk) as Array; for (const row of rows) { out.push({ ...rowToEdge(row), edgeId: row.id, sourceFilePath: row.source_file_path, sourceLanguage: row.source_language, }); } } return out; } /** Delete edges by primary key — the rebind pass's half of a re-resolution. */ deleteEdgesByIds(edgeIds: number[]): number { if (edgeIds.length === 0) return 0; let changed = 0; this.db.transaction(() => { for (let i = 0; i < edgeIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = edgeIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); changed += this.db.prepare(`DELETE FROM edges WHERE id IN (${placeholders})`).run(...chunk).changes; } })(); return changed; } /** * Distinct node names present in the given files — the symbol names a sync * pass uses to look up retryable failed refs after those files changed. */ getNodeNamesByFiles(filePaths: string[]): string[] { if (filePaths.length === 0) return []; const names = new Set(); for (let i = 0; i < filePaths.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = filePaths.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); const rows = this.db .prepare(`SELECT DISTINCT name FROM nodes WHERE file_path IN (${placeholders})`) .all(...chunk) as Array<{ name: string }>; for (const row of rows) names.add(row.name); } return [...names]; } /** * Distinct `file\0name` pairs defined by the given files — the shape sync's * definition delta needs (CG-33). * * Deliberately NOT `getNodeNamesByFiles`: a bare name set is taken over the * WHOLE changed batch, so a name that moves between two files in one commit * (or exists in one changed file and is newly added to another) appears on * both sides and cancels out of the symmetric difference — even though a * definition genuinely appeared or vanished and every reference to that name * repo-wide may now bind elsewhere. Keying by file makes each definition its * own fact, so the move is seen as one removal plus one addition. */ getNodeNamePairsByFiles(filePaths: string[]): Set { const pairs = new Set(); if (filePaths.length === 0) return pairs; for (let i = 0; i < filePaths.length; i += SQLITE_PARAM_CHUNK_SIZE) { const chunk = filePaths.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(','); const rows = this.db .prepare(`SELECT DISTINCT file_path, name FROM nodes WHERE file_path IN (${placeholders})`) .all(...chunk) as Array<{ file_path: string; name: string }>; // NUL-joined: a path or a symbol name can contain a space, never a NUL. for (const row of rows) pairs.add(`${row.file_path}\0${row.name}`); } return pairs; } // =========================================================================== // Statistics // =========================================================================== /** * Lightweight (nodes, edges) count snapshot. Used around an index/sync * run to compute true additions across extraction + resolution + * synthesis — the per-phase counter in the orchestrator only sees * extraction's contribution, which is why the CLI summary under-reported * the edge count (resolution + synthesizer edges were invisible). */ getNodeAndEdgeCount(): { nodes: number; edges: number } { return this.db .prepare('SELECT (SELECT COUNT(*) FROM nodes) AS nodes, (SELECT COUNT(*) FROM edges) AS edges') .get() as { nodes: number; edges: number }; } /** * Get graph statistics */ getStats(): GraphStats { // Single query for all three aggregate counts const counts = this.db.prepare(` SELECT (SELECT COUNT(*) FROM nodes) AS node_count, (SELECT COUNT(*) FROM edges) AS edge_count, (SELECT COUNT(*) FROM files) AS file_count `).get() as { node_count: number; edge_count: number; file_count: number }; const nodesByKind = {} as Record; const nodeKindRows = this.db .prepare('SELECT kind, COUNT(*) as count FROM nodes GROUP BY kind') .all() as Array<{ kind: string; count: number }>; for (const row of nodeKindRows) { nodesByKind[row.kind as NodeKind] = row.count; } const edgesByKind = {} as Record; const edgeKindRows = this.db .prepare('SELECT kind, COUNT(*) as count FROM edges GROUP BY kind') .all() as Array<{ kind: string; count: number }>; for (const row of edgeKindRows) { edgesByKind[row.kind as EdgeKind] = row.count; } const filesByLanguage = {} as Record; const languageRows = this.db .prepare('SELECT language, COUNT(*) as count FROM files GROUP BY language') .all() as Array<{ language: string; count: number }>; for (const row of languageRows) { filesByLanguage[row.language as Language] = row.count; } return { nodeCount: counts.node_count, edgeCount: counts.edge_count, fileCount: counts.file_count, nodesByKind, edgesByKind, filesByLanguage, dbSizeBytes: 0, // Set by caller using DatabaseConnection.getSize() walSizeBytes: 0, // Set by caller using DatabaseConnection.getWalSizeBytes() lastUpdated: Date.now(), }; } // =========================================================================== // Project Metadata // =========================================================================== /** * Get a metadata value by key */ getMetadata(key: string): string | null { const row = this.db.prepare('SELECT value FROM project_metadata WHERE key = ?').get(key) as { value: string } | undefined; return row?.value ?? null; } /** * Set a metadata key-value pair (upsert) */ setMetadata(key: string, value: string): void { this.db.prepare( 'INSERT INTO project_metadata (key, value, updated_at) VALUES (?, ?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at' ).run(key, value, Date.now()); } /** * Get all metadata as a key-value record */ getAllMetadata(): Record { const rows = this.db.prepare('SELECT key, value FROM project_metadata').all() as { key: string; value: string }[]; const result: Record = {}; for (const row of rows) { result[row.key] = row.value; } return result; } /** * Clear all data from the database */ clear(): void { this.nodeCache.clear(); this.db.transaction(() => { this.db.exec('DELETE FROM unresolved_refs'); this.db.exec('DELETE FROM edges'); this.db.exec('DELETE FROM nodes'); this.db.exec('DELETE FROM files'); })(); } }