codegraph/src/extraction/index.ts

/**
 * Extraction Orchestrator
 *
 * Coordinates file scanning, parsing, and database storage.
 */

import * as fs from 'fs';
import * as fsp from 'fs/promises';
import * as path from 'path';
import * as os from 'os';
import * as crypto from 'crypto';
import { execFileSync } from 'child_process';
import {
  Language,
  FileRecord,
  ExtractionResult,
  ExtractionError,
  Node,
  Edge,
  UnresolvedReference,
  ReferenceKind,
} from '../types';
import { QueryBuilder } from '../db/queries';
import { extractFromSource } from './tree-sitter';
import { ParseWorkerPool, resolveParsePoolSize, resolveParseTimeoutMs } from './parse-pool';
import { StoreWriter, StoreBundle, finalizeStoreBundle } from './store-writer';
import { materializeKernelResult } from './kernel';
import { detectGeneratedFile } from './generated-detection';
import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes } from './grammars';
import { loadExtensionOverrides, loadIncludeIgnoredPatterns, loadExcludePatterns, loadIncludePatterns, PROJECT_CONFIG_FILENAME } from '../project-config';
import { isCodeGraphDataDir } from '../directory';
import { logDebug, logWarn } from '../errors';
import { validatePathWithinRoot, normalizePath } from '../utils';
import ignore, { Ignore } from 'ignore';
import { detectFrameworks } from '../resolution/frameworks';
import type { ResolutionContext } from '../resolution/types';
import { createYielder, type MaybeYield } from '../resolution/cooperative-yield';

/**
 * Number of files to read in parallel during indexing.
 * File reads are I/O-bound; batching overlaps I/O wait with CPU parse work.
 */
const FILE_IO_BATCH_SIZE = 10;

/**
 * How many files the `sync()` reconcile processes between cooperative yields to
 * the event loop. The reconcile runs two O(files) loops of synchronous `fs`
 * calls (existsSync for removals, statSync for adds/mods); on a very large repo
 * (~100k files) an un-yielded run wedges the main thread for minutes, which both
 * trips the liveness watchdog (it SIGKILLs a process whose loop stops turning)
 * and blocks the first MCP tool call behind the catch-up gate (issue #905).
 * Yielding every N files keeps the socket, the watchdog heartbeat, and any
 * concurrent read query responsive while the reconcile runs.
 */
const SYNC_RECONCILE_YIELD_INTERVAL = 1000;

// PARSER_RESET_INTERVAL moved to parse-worker.ts (runs in worker thread)

/**
 * Maximum time (ms) to wait for a single file to parse in the worker thread.
 * If tree-sitter hangs or WASM runs out of memory, this prevents the entire
 * indexing run from freezing. The worker is restarted after a (hard) timeout.
 * Env-overridable via CODEGRAPH_PARSE_TIMEOUT_MS for slow storage (#1231).
 */
const PARSE_TIMEOUT_MS = resolveParseTimeoutMs(process.env.CODEGRAPH_PARSE_TIMEOUT_MS);

/**
 * Number of files to parse before recycling the worker thread.
 * WASM linear memory can grow but NEVER shrink (WebAssembly spec limitation).
 * The only way to reclaim tree-sitter's WASM heap is to destroy the entire
 * V8 isolate by terminating the worker thread and spawning a fresh one.
 * This interval balances memory usage against the cost of reloading grammars.
 */
const WORKER_RECYCLE_INTERVAL = 250;

/**
 * Progress callback for indexing operations
 */
export interface IndexProgress {
  phase: 'scanning' | 'parsing' | 'storing' | 'resolving' | 'linking';
  current: number;
  total: number;
  currentFile?: string;
}

/**
 * Result of an indexing operation
 */
export interface IndexResult {
  success: boolean;
  filesIndexed: number;
  filesSkipped: number;
  filesErrored: number;
  /**
   * How many indexable files the scan discovered — the ground truth the
   * indexed/skipped/errored tallies must add up to. A shortfall means files
   * were silently dropped mid-pipeline (e.g. a killed worker under load) and
   * the index is PARTIAL; callers surface that rather than trusting the
   * counts. Only set by full-index runs (indexAll), not indexFiles/sync.
   */
  filesDiscovered?: number;
  /**
   * Files the scan saw but has no grammar for, tallied by extension. Only the
   * degenerate case needs it: a project of unsupported files otherwise looks
   * exactly like an empty one (0 files, state `complete`), so nothing tells the
   * user — or an agent — that there was code here CodeGraph could not read
   * (#1502). Counted during the scan's existing walk.
   */
  filesSkippedUnsupported?: number;
  /** The most common unsupported extensions, biggest first. */
  topUnsupportedExtensions?: { ext: string; count: number }[];
  nodesCreated: number;
  edgesCreated: number;
  errors: ExtractionError[];
  durationMs: number;
}

/**
 * Result of a sync operation
 */
export interface SyncResult {
  filesChecked: number;
  filesAdded: number;
  filesModified: number;
  filesRemoved: number;
  nodesUpdated: number;
  durationMs: number;
  changedFilePaths?: string[];
  /** Paths not absorbed because reading or extraction failed; retain for status/retry. */
  failedFilePaths?: string[];
  /**
   * Symbol names whose set of definitions this sync CHANGED — names the synced
   * files gained or lost, as the symmetric difference of their `file\0name`
   * definition pairs before and after the store phase (per file, so a name
   * moving between two changed files does not cancel itself out).
   * Resolution picks among all same-named definitions project-wide,
   * so these are exactly the names whose already-resolved edges — in files this
   * sync never touched — may now bind elsewhere and must be re-resolved for the
   * index to stay convergent with a full rebuild (CG-33).
   *
   * A body-only edit leaves this empty, which is the common case and costs
   * nothing downstream.
   */
  definitionDelta?: string[];
}

/**
 * Calculate SHA256 hash of file contents
 */
export function hashContent(content: string): string {
  return crypto.createHash('sha256').update(content).digest('hex');
}

/**
 * Skip files larger than this (bytes). Generated bundles, minified JS, and
 * vendored blobs blow the WASM heap and the worker-recycle budget for no useful
 * symbols. 1 MB covers essentially all hand-written source.
 */
const MAX_FILE_SIZE = 1024 * 1024;

/**
 * Directory names that are dependency, build, cache, or tooling output across the
 * languages/frameworks CodeGraph supports — curated from the canonical
 * github/gitignore templates. Excluded by default so the graph reflects your code,
 * not third-party noise, without requiring a `.gitignore` (issue #407). The
 * exclusion applies uniformly (git or not, tracked or not); the only opt-in is an
 * explicit `.gitignore` negation (e.g. `!vendor/`). First-party-prone or generic
 * names (`packages`, `lib`, `app`, `bin`, `src`, `deps`, `env`, `tmp`, `storage`,
 * `Library`) are deliberately NOT listed, to avoid ever hiding real source.
 *
 * Only dirs that actually contain *indexable source* (or are enormous) earn a slot
 * — IDE/state dirs like `.idea`/`.vs` are omitted because CodeGraph indexes only
 * recognized source extensions, so they produce no symbols regardless.
 */
const DEFAULT_IGNORE_DIRS: ReadonlySet<string> = new Set([
  // JS / TS — dependency directories
  'node_modules', 'bower_components', 'jspm_packages', 'web_modules',
  '.yarn', '.pnpm-store',
  // JS / TS — framework & bundler build / cache / deploy output
  '.next', '.nuxt', '.svelte-kit', '.turbo', '.vite', '.parcel-cache', '.angular',
  '.docusaurus', 'storybook-static', '.vinxi', '.nitro', 'out-tsc',
  '.vercel', '.netlify', '.wrangler',
  // Build output (common across ecosystems)
  'dist', 'build', 'out', '.output',
  // Test / coverage
  'coverage', '.nyc_output',
  // Python
  '__pycache__', '__pypackages__', '.venv', 'venv', '.pixi', '.pdm-build',
  '.mypy_cache', '.pytest_cache', '.ruff_cache', '.tox', '.nox', '.hypothesis',
  '.ipynb_checkpoints', '.eggs',
  // Rust / JVM (Maven, Gradle, Scala)
  'target', '.gradle',
  // .NET
  'obj',
  // Vendored deps (Go, PHP/Composer, Ruby/Bundler)
  'vendor',
  // Swift / iOS
  '.build', 'Pods', 'Carthage', 'DerivedData', '.swiftpm',
  // Dart / Flutter
  '.dart_tool', '.pub-cache',
  // Native (Android NDK, C/C++ deps)
  '.cxx', '.externalNativeBuild', 'vcpkg_installed',
  // Scala tooling
  '.bloop', '.metals',
  // Lua / Luau (LuaRocks)
  'lua_modules', '.luarocks',
  // Delphi / RAD Studio IDE backups (duplicate .pas source — would double-count)
  '__history', '__recovery',
  // Generic cache
  '.cache',
]);

/**
 * Android resource directory types. A `res/` tree holds ONLY non-code resources —
 * layouts, drawables, value bags (strings/colors/styles), menus, navigation
 * graphs — split into one typed subdirectory per kind, optionally density/locale/
 * version-qualified (`values-es`, `drawable-hdpi`, `layout-v21`, …). None of it
 * yields an extractable code symbol, yet on an Android app it DOMINATES the tree
 * (one report: 26k XML files = 97% of the project, 0 symbols), bloating the DB,
 * slowing indexing, and skewing both the file count and `codegraph_explore`
 * results (#1047). So these are excluded by default. The structure is
 * self-identifying — a non-Android project has no `res/layout/` etc., so it's
 * untouched — and the only XML that DOES produce symbols (MyBatis mappers) lives
 * under `src/main/resources/`, never `res/`, so nothing useful is dropped.
 * `res/raw/` is deliberately NOT here: it holds arbitrary bundled assets that can
 * be code-ish (a `.sql` schema, a `.js`), so we leave it indexed. Override any of
 * these with a `.gitignore` negation (e.g. `!res/values/`).
 */
const ANDROID_RES_TYPES: readonly string[] = [
  'anim', 'animator', 'color', 'drawable', 'font', 'layout',
  'menu', 'mipmap', 'navigation', 'transition', 'values', 'xml',
];

/** Gitignore-style patterns for the `ignore` matcher: the dirs above plus a few globs. */
const DEFAULT_IGNORE_PATTERNS: string[] = [
  ...Array.from(DEFAULT_IGNORE_DIRS, (d) => `${d}/`),
  '*.egg-info/',     // Python packaging metadata
  'cmake-build-*/',  // CLion / CMake build trees
  'bazel-*/',        // Bazel output symlink trees
  // Android resource dirs at any depth, with their qualifier variants (#1047).
  ...ANDROID_RES_TYPES.map((t) => `**/res/${t}*/`),
];

/** True if `buf` decodes as strict UTF-8 (no invalid byte sequences). */
function isValidUtf8(buf: Buffer): boolean {
  try {
    new TextDecoder('utf-8', { fatal: true }).decode(buf);
    return true;
  } catch {
    return false;
  }
}

/**
 * Read a `.gitignore` and return patterns safe to hand to the `ignore` matcher —
 * never throwing, even when the file isn't real gitignore text. Two failure
 * modes, both seen in the wild (issue #682):
 *
 *  - The file isn't valid UTF-8 — e.g. transparently encrypted in place by
 *    corporate DLP / endpoint-security software, leaving a UTF-16 header plus
 *    ciphertext. None of it is meaningful patterns, so the whole file is skipped.
 *  - The file is text but a single line can't be compiled to a regex by the
 *    `ignore` library — `\\[` and friends throw "Unterminated character class".
 *    Crucially the throw is LAZY (at match time, not `.add()`), so it would
 *    otherwise escape mid-scan. That one pattern is dropped; the rest are kept.
 *
 * Either way a warning that NAMES the file is logged (the reporter couldn't tell
 * which `.gitignore` was at fault) and indexing continues instead of aborting.
 * Returns '' when there's nothing usable.
 */
function readGitignorePatterns(giPath: string): string {
  let buf: Buffer;
  try {
    buf = fs.readFileSync(giPath);
  } catch {
    return ''; // unreadable (permissions / race) — treat as absent
  }
  // A NUL byte never appears in real gitignore text, and a fatal UTF-8 decode
  // catches the rest. Such a file isn't ignore patterns at all.
  if (buf.includes(0) || !isValidUtf8(buf)) {
    logWarn(
      'Ignoring a .gitignore that is not valid UTF-8 text — it may have been encrypted ' +
        'in place by endpoint-security software. Indexing continues without it.',
      { file: giPath },
    );
    return '';
  }
  const content = buf.toString('utf-8');
  // Fast path: one `.ignores()` call forces the library to compile EVERY rule,
  // so if it doesn't throw, the whole file is safe to use verbatim.
  try {
    ignore().add(content).ignores('.codegraph-probe');
    return content;
  } catch {
    // Fall through: a line is uncompilable — keep the good ones, drop the bad.
  }
  const kept: string[] = [];
  let dropped = 0;
  for (const line of content.split(/\r?\n/)) {
    try {
      ignore().add(line).ignores('.codegraph-probe');
      kept.push(line);
    } catch {
      dropped++;
    }
  }
  if (dropped > 0) {
    logWarn(
      `Skipped ${dropped} unparseable pattern(s) in a .gitignore; the rest are applied.`,
      { file: giPath },
    );
  }
  return kept.join('\n');
}

/**
 * Resolve the repository GIT_DIR for `repoRoot` (a `.git` directory, or the
 * target of a `.git` file pointer). Null when this isn't a git checkout.
 */
function resolveGitDir(repoRoot: string): string | null {
  const gitPath = path.join(repoRoot, '.git');
  let st: fs.Stats;
  try {
    st = fs.statSync(gitPath);
  } catch {
    return null;
  }
  if (st.isDirectory()) return gitPath;
  if (!st.isFile()) return null;
  try {
    const raw = fs.readFileSync(gitPath, 'utf8').match(/^gitdir:\s*(.+)$/m)?.[1]?.trim();
    if (!raw) return null;
    return path.isAbsolute(raw) ? path.normalize(raw) : path.resolve(repoRoot, raw);
  } catch {
    return null;
  }
}

/** Expand a leading `~/` the way git does for `core.excludesFile`. */
function expandUserPath(p: string): string {
  if (p === '~') return os.homedir();
  if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2));
  return p;
}

/**
 * Root-relative exclude patterns from git sources that are NOT the root
 * `.gitignore`: `.git/info/exclude` and `core.excludesFile`. Same semantics as
 * the root `.gitignore`, so they merge into {@link buildDefaultIgnore}. Without
 * these, the watcher / FS-walk scope silently diverged from
 * `git ls-files --exclude-standard` (#1728).
 */
function readGitExcludeExtraPatterns(rootDir: string): string {
  const chunks: string[] = [];
  const gitDir = resolveGitDir(rootDir);
  if (gitDir) {
    const excludePath = path.join(gitDir, 'info', 'exclude');
    if (fs.existsSync(excludePath)) {
      const patterns = readGitignorePatterns(excludePath);
      if (patterns) chunks.push(patterns);
    }
  }
  try {
    const configured = execFileSync(
      'git',
      ['-C', rootDir, 'config', '--get', 'core.excludesFile'],
      { encoding: 'utf8', timeout: 5_000, stdio: ['ignore', 'pipe', 'ignore'] },
    ).trim();
    if (configured) {
      const abs = expandUserPath(configured);
      if (fs.existsSync(abs)) {
        const patterns = readGitignorePatterns(abs);
        if (patterns) chunks.push(patterns);
      }
    }
  } catch {
    // No git, unset, or timeout — leave extras empty.
  }
  return chunks.join('\n');
}

/**
 * Directories `git ls-files -o -i --exclude-standard --directory` reports as
 * ignored-untracked. Seeded into {@link ScopeIgnore} so nested `.gitignore`
 * effects (and any exclude-standard rule the flat matcher might miss) prune the
 * watcher the same way the indexer skips them (#1728).
 */
function listGitIgnoredDirectories(rootDir: string): string[] {
  try {
    const out = execFileSync(
      'git',
      ['-C', rootDir, 'ls-files', '-z', '-o', '-i', '--exclude-standard', '--directory'],
      {
        encoding: 'utf8',
        timeout: 60_000,
        maxBuffer: 50 * 1024 * 1024,
        stdio: ['ignore', 'pipe', 'ignore'],
      },
    );
    const dirs: string[] = [];
    for (const entry of out.split('\0')) {
      if (!entry) continue;
      dirs.push(entry.endsWith('/') ? entry : `${entry}/`);
    }
    return dirs;
  } catch {
    return [];
  }
}

/**
 * An `ignore` matcher seeded with the built-in defaults, merged with the project's
 * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default, plus
 * git's other root-relative exclude files (`.git/info/exclude`, `core.excludesFile`)
 * so watcher / FS-walk scope matches `git ls-files --exclude-standard` (#1728).
 * Shared by both enumeration paths so behavior is identical with or without git —
 * and so the defaults apply to tracked files too (committing a dependency dir
 * doesn't make it project code; the explicit `.gitignore` negation is the only
 * opt-in).
 */
export function buildDefaultIgnore(rootDir: string): Ignore {
  const ig = ignore().add(DEFAULT_IGNORE_PATTERNS);
  const rootGitignore = path.join(rootDir, '.gitignore');
  if (fs.existsSync(rootGitignore)) ig.add(readGitignorePatterns(rootGitignore));
  const extra = readGitExcludeExtraPatterns(rootDir);
  if (extra) ig.add(extra);
  return ig;
}

/**
 * Defaults-only ignore matcher (no root `.gitignore` merged). Used wherever the
 * parent repo's own ignore rules must NOT apply — inside embedded child repos,
 * whose gitignore semantics their own `git ls-files` already enforced (#514).
 */
function defaultsOnlyIgnore(): Ignore {
  return ignore().add(DEFAULT_IGNORE_PATTERNS);
}

/**
 * Matcher for the project's `codegraph.json` `includeIgnored` patterns — the
 * explicit opt-in to index embedded git repos living inside gitignored
 * directories (#622, #699). Returns `null` when the project opted in nothing,
 * which is the zero-config DEFAULT: `.gitignore` is then fully respected and a
 * gitignored directory (even one holding nested repos) is never walked or
 * indexed (#970, #976). Built once per scan/sync/scope operation from the scan
 * root and threaded down — never global, so multi-project daemons stay isolated.
 */
function loadIncludeIgnoredMatcher(rootDir: string): Ignore | null {
  const patterns = loadIncludeIgnoredPatterns(rootDir);
  return patterns.length > 0 ? ignore().add(patterns) : null;
}

/**
 * Matcher for the project's `codegraph.json` `exclude` patterns — paths to keep
 * OUT of the index even when git-tracked, which `.gitignore` cannot do (#999).
 * The escape hatch for a committed vendor/theme/SDK directory. Returns `null`
 * when nothing is excluded (the zero-config default → no overhead). Matched
 * against project-root-relative paths, so it applies uniformly across the whole
 * workspace, including inside embedded repos (excluding `static/` means gone
 * everywhere). Built once per scan/sync/scope operation from the scan root.
 */
function loadExcludeMatcher(rootDir: string): Ignore | null {
  const patterns = loadExcludePatterns(rootDir);
  return patterns.length > 0 ? ignore().add(patterns) : null;
}

/**
 * Matcher for the project's `codegraph.json` `include` patterns — first-party
 * source to force INTO the index even when `.gitignore` drops it (the general
 * whitelist `includeIgnored` never was — that one only revives *embedded git
 * repos*). The case it exists for: a project under a second VCS (SVN/Perforce)
 * `.gitignore`s its own real source so it stays out of Git, yet we still want it
 * indexed. Returns `null` when nothing is force-included (the zero-config
 * default → no overhead, no extra walk). Built once per scan/sync/scope
 * operation from the scan root.
 */
function loadIncludeMatcher(rootDir: string): Ignore | null {
  const patterns = loadIncludePatterns(rootDir);
  return patterns.length > 0 ? ignore().add(patterns) : null;
}

/** Glob metacharacters that end the static (literal) prefix of an `include` pattern. */
const GLOB_META = /[*?[\]{}!]/;

/**
 * The static directory prefix of each `include` pattern — the literal leading
 * path up to the first glob segment — trailing-slashed, used to (a) walk only
 * the opted-in subtrees in `collectIncludedFiles` and (b) let `ScopeIgnore` keep
 * the watcher descending toward them. `Tools/` stays `Tools/`; a recursive
 * `Tools/**` glob yields `Tools/`; `src/local/file.ts` yields `src/local/` (the
 * file's dir); a pattern that starts with a glob (like a leading `**`) yields
 * `''`, meaning "no static root — walk the whole tree". Duplicates and roots
 * nested under a broader root are collapsed so each subtree is walked once.
 */
function includeStaticRoots(patterns: string[]): string[] {
  const roots = new Set<string>();
  for (const pattern of patterns) {
    let p = pattern.replace(/^\/+/, '');
    const trailingSlash = p.endsWith('/');
    if (trailingSlash) p = p.slice(0, -1);
    const segs = p.split('/').filter(Boolean);
    const lead: string[] = [];
    for (const s of segs) {
      if (GLOB_META.test(s)) break;
      lead.push(s);
    }
    const hadWildcard = lead.length < segs.length;
    // A wholly-literal pattern with no trailing slash names a file (or a dir we
    // can't tell apart) — drop its last segment so we walk the containing dir
    // and let the matcher pick the file. A trailing slash or a glob means the
    // remaining `lead` is already the directory to walk.
    if (!hadWildcard && !trailingSlash && lead.length > 0) lead.pop();
    if (lead.length === 0) {
      roots.clear();
      roots.add('');
      return ['']; // a top-level glob forces a whole-tree walk; nothing narrower matters
    }
    roots.add(lead.join('/') + '/');
  }
  // Collapse roots nested under a broader one (e.g. drop `a/b/` if `a/` is present).
  const all = [...roots];
  return all.filter((r) => !all.some((other) => other !== r && r.startsWith(other)));
}

/**
 * Actively discover the source files an `include` whitelist forces in. `git
 * ls-files` never lists gitignored files, so a filtered filesystem walk of just
 * the opted-in subtrees (`includeStaticRoots`) is the only way to find them.
 * Returns project-root-relative, normalized source-file paths.
 *
 * A file is collected when it MATCHES `include`, is NOT hit by `exclude` (an
 * explicit exclude always wins), is a recognized source file, and does not live
 * under a built-in default-ignored dir (`node_modules`, `dist`, …), `.git`, or
 * CodeGraph's data dir — those are never resurfaced, mirroring `ScopeIgnore`.
 * `.gitignore` is deliberately NOT consulted: overriding it is the whole point.
 */
function collectIncludedFiles(
  rootDir: string,
  include: Ignore,
  exclude: Ignore | null,
  roots: string[],
  overrides: Record<string, Language>,
): Set<string> {
  const out = new Set<string>();
  const defaults = defaultsOnlyIgnore();
  const visited = new Set<string>();

  const consider = (abs: string, rel: string, isDir: boolean): void => {
    if (isDir) {
      if (defaults.ignores(rel + '/')) return; // never node_modules/dist/… via include
      // An explicit `exclude` always wins over `include`; prune the whole subtree
      // here so a large excluded dir (a committed frontend's own vendored deps,
      // build output, …) is never walked — the per-file guard below still catches
      // anything a directory pattern doesn't, so this is a pure efficiency win.
      if (exclude && exclude.ignores(rel + '/')) return;
      walk(abs);
    } else {
      if (defaults.ignores(rel)) return;
      if (!include.ignores(rel)) return;
      if (exclude && exclude.ignores(rel)) return;
      if (!isSourceFile(rel, overrides)) return;
      out.add(rel);
    }
  };

  function walk(absDir: string): void {
    let realDir: string;
    try {
      realDir = fs.realpathSync(absDir);
    } catch {
      return;
    }
    if (visited.has(realDir)) return; // symlink-cycle guard
    visited.add(realDir);

    let entries: fs.Dirent[];
    try {
      entries = fs.readdirSync(absDir, { withFileTypes: true });
    } catch {
      return;
    }
    for (const entry of entries) {
      if (entry.name === '.git' || isCodeGraphDataDir(entry.name)) continue;
      const abs = path.join(absDir, entry.name);
      const rel = normalizePath(path.relative(rootDir, abs));
      if (!rel || rel.startsWith('..')) continue;
      if (entry.isSymbolicLink()) {
        try {
          const st = fs.statSync(fs.realpathSync(abs));
          consider(abs, rel, st.isDirectory());
        } catch {
          // broken symlink — skip
        }
        continue;
      }
      consider(abs, rel, entry.isDirectory());
    }
  }

  for (const root of roots) {
    walk(root === '' ? rootDir : path.join(rootDir, root));
  }
  return out;
}

/**
 * The included source files (`codegraph.json` `include`) for a scan root, or an
 * empty set when nothing is force-included. Centralizes loading the matcher,
 * roots, exclude, and overrides so both enumeration paths (git and filesystem
 * walk) add the same files.
 */
function collectIncludedFilesForRoot(rootDir: string): Set<string> {
  const include = loadIncludeMatcher(rootDir);
  if (!include) return new Set();
  const roots = includeStaticRoots(loadIncludePatterns(rootDir));
  return collectIncludedFiles(rootDir, include, loadExcludeMatcher(rootDir), roots, loadExtensionOverrides(rootDir));
}

/**
 * `git ls-files --directory` collapses a wholly-untracked/ignored directory into
 * one entry — and when the command's own cwd is such a directory (the indexed
 * root is itself a git-ignored subdir of an enclosing repo), git emits the
 * literal `./` meaning "this entire directory". That sentinel is not a real
 * nested path: feeding it to the `ignore` matcher throws ("path should be a
 * `path.relative()`d string, but got "./""), which used to abort `buildScopeIgnore`
 * and so break the MCP daemon's watcher/auto-sync on connect; and joining it back
 * onto `repoDir` would just re-point at the cwd. Drop it wherever we consume
 * `--directory` output. (#936)
 */
function isWholeCwdEntry(entry: string): boolean {
  return entry === './' || entry === '.' || entry === '';
}

/**
 * List the gitignored DIRECTORIES of a repo (collapsed, trailing-slash form),
 * relative to `repoDir`. These are invisible to every other `git ls-files` /
 * `git status` mode — and in a multi-repo workspace they are exactly where the
 * nested project repos live (a super-repo `.gitignore`s its child repos to keep
 * `git status` quiet; that does not make them third-party code). (#514)
 */
function listIgnoredDirs(repoDir: string): string[] {
  try {
    const out = execFileSync(
      'git',
      ['ls-files', '-z', '-o', '-i', '--exclude-standard', '--directory'],
      { cwd: repoDir, encoding: 'utf-8' as const, timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] as ['pipe', 'pipe', 'pipe'], windowsHide: true }
    );
    return out.split('\0').filter((e) => e.endsWith('/') && !isWholeCwdEntry(e));
  } catch {
    return [];
  }
}

/** Max directory depth searched below an ignored dir for nested `.git` roots. */
const EMBEDDED_REPO_SEARCH_DEPTH = 4;
/** Max directories examined per search — a huge ignored data dir must never stall a scan/sync. */
const EMBEDDED_REPO_SEARCH_ENTRIES = 2000;

/**
 * Classify a directory's `.git` entry for embedded-repo discovery.
 *
 * - A `.git` **directory** is an embedded clone — distinct first-party code a
 *   super-repo merely hides from git; index it (#193, #514).
 * - A `.git` **file** is a pointer (`gitdir: …`). A git **worktree** points into
 *   the host repo's own `.git/worktrees/<name>`, so it is a second working view
 *   of a repo CodeGraph already indexes — indexing it just duplicates the whole
 *   graph N times; skip it (#848). A **submodule worktree** points into
 *   `.git/modules/<module>/worktrees/<name>` — same duplication, so skip it too
 *   (#945). A **submodule** checkout points into `.git/modules/<module>` (no
 *   `worktrees/` segment) and is distinct code, so index it as before.
 *
 * Returns `'none'` when there is no `.git` entry here.
 */
function classifyGitDir(absDir: string): 'embedded' | 'worktree' | 'none' {
  let st: fs.Stats;
  try {
    st = fs.statSync(path.join(absDir, '.git'));
  } catch {
    return 'none';
  }
  if (st.isDirectory()) return 'embedded';
  if (!st.isFile()) return 'none';
  try {
    const gitdir = fs.readFileSync(path.join(absDir, '.git'), 'utf8').match(/^gitdir:\s*(.+)$/m)?.[1]?.trim();
    // A worktree's gitdir lives under some repo's `.git/worktrees/<name>` —
    // either the top-level repo's (`.git/worktrees/`) or, for a worktree of a
    // submodule, that submodule's gitdir (`.git/modules/<module>/worktrees/`).
    // The optional `modules/<module>` segment covers the submodule case (#945).
    // Match both separators so a Windows-style pointer is recognized too.
    if (gitdir && /(^|[\\/])\.git[\\/](modules[\\/][^\\/]+[\\/])?worktrees[\\/]/.test(gitdir)) return 'worktree';
  } catch {
    // Unreadable `.git` pointer — fall back to the prior "index it" behavior.
  }
  return 'embedded';
}

/**
 * Find git repositories nested under `absDir` (inclusive), shallow bounded BFS.
 * Stops descending at each repo root found — contents belong to that repo's own
 * enumeration. Skips default-ignored dirs (`node_modules` can contain `.git`
 * from npm git-dependencies — that never makes it project code) and CodeGraph
 * data dirs. Depth- and entry-capped so a huge ignored tree can't stall the scan.
 */
function findNestedGitRepos(absDir: string, relPrefix: string): string[] {
  const found: string[] = [];
  const defaults = defaultsOnlyIgnore();
  const queue: Array<{ abs: string; rel: string; depth: number }> = [
    { abs: absDir, rel: relPrefix, depth: 0 },
  ];
  let examined = 0;
  while (queue.length > 0) {
    const { abs, rel, depth } = queue.shift()!;
    if (++examined > EMBEDDED_REPO_SEARCH_ENTRIES) {
      logDebug('Embedded-repo search entry cap hit — deeper repos (if any) not discovered', { under: relPrefix });
      break;
    }
    const cls = classifyGitDir(abs);
    if (cls === 'worktree') {
      continue; // a git worktree duplicates an already-indexed repo (#848) — skip
    }
    if (cls === 'embedded') {
      found.push(rel);
      continue; // its own git handles everything below
    }
    if (depth >= EMBEDDED_REPO_SEARCH_DEPTH) continue;
    let entries: fs.Dirent[];
    try {
      entries = fs.readdirSync(abs, { withFileTypes: true });
    } catch {
      continue;
    }
    for (const entry of entries) {
      if (!entry.isDirectory()) continue;
      if (entry.name === '.git' || isCodeGraphDataDir(entry.name)) continue;
      const childRel = rel + entry.name + '/';
      if (defaults.ignores(childRel)) continue;
      queue.push({ abs: path.join(abs, entry.name), rel: childRel, depth: depth + 1 });
    }
  }
  return found;
}

/**
 * Workspace-scope ignore matcher. Ordinary paths get the root's matcher
 * (built-in defaults + root `.gitignore` + `.git/info/exclude` +
 * `core.excludesFile`, plus directories `git ls-files --exclude-standard`
 * reports as ignored); paths inside an EMBEDDED repo get that repo's own
 * matcher — the parent's `.gitignore` hides a child repo from git, not from
 * the index (#514). A directory path (trailing slash) that is an ANCESTOR of
 * an embedded root is never ignored, so directory-pruning callers (the Linux
 * per-directory watcher) still descend to reach the embedded repos.
 *
 * Shared by the indexer (scoped sync / skip checks) and the watcher so their
 * scope cannot diverge from each other or from `git ls-files --exclude-standard`
 * (#1728).
 */

/**
 * The grammars to preload for a file set.
 *
 * Path-only detection calls every `.h` file C, but parse-time detection reads
 * the source and can reclassify it as C++ or Objective-C (`detectLanguage`
 * with a `source` argument). Workers only ever get the grammars named here, so
 * a header that turns out to be Objective-C in a project with no `.m` file
 * found no parser and failed with `Failed to get parser for language: objc`
 * (#1628). C++ was already covered; Objective-C was not.
 */
export function preloadLanguagesForFiles(
  files: string[],
  overrides?: Record<string, Language>
): Language[] {
  const languages = [...new Set(files.map((f) => detectLanguage(f, undefined, overrides)))];
  if (languages.includes('c')) {
    for (const ambiguous of ['cpp', 'objc'] as const) {
      if (!languages.includes(ambiguous)) languages.push(ambiguous);
    }
  }
  return languages;
}

export class ScopeIgnore {
  private embedded: Array<{ root: string; matcher: Ignore }>;
  private defaults: Ignore = defaultsOnlyIgnore();
  constructor(
    private rootMatcher: Ignore,
    embedded: Array<{ root: string; matcher: Ignore }>,
    /**
     * Project `codegraph.json` `exclude` patterns (#999), matched against the
     * full root-relative path. Wins over everything else — an explicit user
     * exclude applies even to tracked files and even inside embedded repos.
     */
    private exclude: Ignore | null = null,
    /**
     * Project `codegraph.json` `include` patterns — first-party source forced
     * INTO the index despite `.gitignore`. When a path matches, it is NOT
     * ignored (so the watcher watches it), overriding `.gitignore`/`rootMatcher`
     * — but never `exclude` (checked first) and never a built-in default-ignored
     * dir. `includeRoots` are the static prefixes so a gitignored ANCESTOR
     * directory of an included subtree still isn't pruned by the directory
     * walker/watcher.
     */
    private include: Ignore | null = null,
    private includeRoots: string[] = [],
  ) {
    // Longest root first so paths in nested embedded repos hit the innermost matcher.
    this.embedded = [...embedded].sort((a, b) => b.root.length - a.root.length);
  }

  ignores(rel: string): boolean {
    // User `exclude` (#999) is checked first and against the full root-relative
    // path: it must drop git-TRACKED paths (which `.gitignore` can't) and apply
    // everywhere, including ancestors of embedded repos.
    if (this.exclude && this.exclude.ignores(rel)) return true;
    // User `include`: force first-party source in despite `.gitignore`. Never
    // resurfaces a built-in default-ignored dir (node_modules/dist/…), so an
    // include pattern can't accidentally pull in dependency/build trees.
    if (this.include && !this.defaults.ignores(rel)) {
      if (rel.endsWith('/')) {
        // A directory on (or leading to) an included subtree must stay walkable
        // so the watcher/walker descends to reach the forced-in files.
        if (this.includeRoots.some((r) => r.startsWith(rel) || rel.startsWith(r))) return false;
      } else if (this.include.ignores(rel)) {
        return false;
      }
    }
    for (const { root, matcher } of this.embedded) {
      if (rel.startsWith(root)) {
        const inner = rel.slice(root.length);
        if (inner === '') return false;
        // Built-in defaults apply to the FULL path uniformly (#407) — an
        // embedded repo inside node_modules (an npm git-dependency) must stay
        // excluded even though its own rules wouldn't ignore its files.
        return this.defaults.ignores(rel) || matcher.ignores(inner);
      }
    }
    // Never prune a directory that leads to an embedded repo.
    if (rel.endsWith('/') && this.embedded.some(({ root }) => root.startsWith(rel))) {
      return false;
    }
    return this.rootMatcher.ignores(rel);
  }
}

/**
 * Build the workspace-scope matcher. When the caller already knows the
 * embedded roots (the scanner discovers them during collection), pass them to
 * skip rediscovery; otherwise they're discovered here (the watcher path).
 */
export function buildScopeIgnore(rootDir: string, embeddedRoots?: Iterable<string>): ScopeIgnore {
  const roots = embeddedRoots ? [...embeddedRoots] : discoverEmbeddedRepoRoots(rootDir);
  const include = loadIncludeMatcher(rootDir);
  // Root matcher already has defaults + root `.gitignore` + info/exclude +
  // core.excludesFile. Seed ignored-untracked directories from git so nested
  // `.gitignore` effects prune the watcher identically to the indexer (#1728).
  const rootMatcher = buildDefaultIgnore(rootDir);
  for (const dir of listGitIgnoredDirectories(rootDir)) {
    rootMatcher.add(dir);
  }
  return new ScopeIgnore(
    rootMatcher,
    roots.map((root) => ({ root, matcher: buildDefaultIgnore(path.join(rootDir, root)) })),
    loadExcludeMatcher(rootDir),
    include,
    include ? includeStaticRoots(loadIncludePatterns(rootDir)) : [],
  );
}

/**
 * Whether an embedded repo found as a tracked gitlink (mode 160000, #1031/#1033)
 * must be SKIPPED rather than indexed. A gitlink is tracked, so `.gitignore`
 * can't untrack it — but the discovery passes for it must still honor the same
 * scope rules as every other path, or a gitignored reference/data dir full of
 * `git add`ed clones gets pulled into the index against the user's stated intent
 * (#1065). Two reasons to skip:
 *   1. It sits in a built-in default-ignored location — an npm git-dependency
 *      under `node_modules` is never project code; not even an explicit opt-in
 *      revives it (matches `findIgnoredEmbeddedRepos`).
 *   2. The parent repo's own `.gitignore` covers its path and the project did
 *      NOT opt that path in via `codegraph.json` `includeIgnored`. The gitignore
 *      rule is the user's stated intent to keep that path out of scope, exactly
 *      as for an UNtracked embedded repo — respect it by default, opt back in
 *      with `includeIgnored` (#514, #970, #976).
 * `relDir` is repoDir-relative (trailing-slashed); `prefix` is repoDir's
 * scan-root-relative path so the `includeIgnored` pattern is matched on the full
 * scan-root-relative path. `defaults` is `defaultsOnlyIgnore()` and `repoIgnore`
 * is `buildDefaultIgnore(repoDir)` (defaults + the repo's own `.gitignore`),
 * both passed in so they're built once per repo level rather than per gitlink.
 */
function gitlinkEmbeddedRepoSkipped(
  relDir: string,
  prefix: string,
  defaults: Ignore,
  repoIgnore: Ignore,
  includeIgnored: Ignore | null,
): boolean {
  if (defaults.ignores(relDir)) return true;        // default-ignored — never index, opt-in can't revive
  if (!repoIgnore.ignores(relDir)) return false;    // not ignored at all — index as before (#1031/#1033)
  // Gitignored by the repo's own rules — skip unless the project opted it in.
  return !includeIgnored?.ignores(normalizePath(prefix + relDir));
}

/**
 * Standalone discovery of every embedded repo root under `rootDir` (relative,
 * trailing-slashed) — the untracked kind (#193) always, and the gitignored kind
 * (#514) only for directories the project opted in via `codegraph.json`
 * `includeIgnored` (#622, #699); otherwise `.gitignore` is respected and they
 * are not discovered (#970, #976). Recursive (an embedded repo can embed further
 * repos). Returns [] for non-git roots: the filesystem walk handles nested repos
 * there already.
 */
export function discoverEmbeddedRepoRoots(rootDir: string): string[] {
  try {
    execFileSync('git', ['rev-parse', '--git-dir'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
  } catch {
    return [];
  }
  const out: string[] = [];
  const defaults = defaultsOnlyIgnore();
  const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
  const visit = (repoAbs: string, prefix: string): void => {
    const candidates: string[] = [];
    try {
      const o = execFileSync(
        'git',
        ['ls-files', '-z', '-o', '--exclude-standard', '--directory'],
        { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }
      );
      for (const e of o.split('\0')) {
        if (e.endsWith('/') && !isWholeCwdEntry(e) && !defaults.ignores(e)) {
          candidates.push(...findNestedGitRepos(path.join(repoAbs, e), e));
        }
      }
    } catch { /* untracked listing failed — ignored-side discovery still runs */ }
    // Unexpanded gitlinks (mode 160000) with a real checkout on disk — embedded
    // repos `git add`ed without `.gitmodules`, or submodules not active here. The
    // untracked listing above can't see them (they're tracked), so find them the
    // same way collectGitFiles does, keeping watcher scope == indexer scope.
    // (#1031, #1033)
    try {
      const staged = lsFilesStaged(
        { cwd: repoAbs, encoding: 'utf-8', timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }
      );
      const repoIgnore = buildDefaultIgnore(repoAbs);
      for (const entry of staged.split('\0')) {
        if (!entry || entry.slice(0, 6) !== '160000') continue;
        const tab = entry.indexOf('\t');
        if (tab === -1) continue;
        const rel = entry.slice(tab + 1);
        const relDir = rel.endsWith('/') ? rel : rel + '/';
        // A gitlink under a gitignored path is respected (not indexed) unless the
        // project opted it in — same rule as the untracked-ignored kind (#1065).
        if (gitlinkEmbeddedRepoSkipped(relDir, prefix, defaults, repoIgnore, includeIgnored)) continue;
        if (classifyGitDir(path.join(repoAbs, rel)) === 'embedded') candidates.push(relDir);
      }
    } catch { /* staged listing failed — other discovery still runs */ }
    candidates.push(...findIgnoredEmbeddedRepos(repoAbs, includeIgnored, prefix));
    for (const rel of candidates) {
      const full = normalizePath(prefix + rel);
      out.push(full);
      visit(path.join(repoAbs, rel), full);
    }
  };
  visit(rootDir, '');
  return out;
}

/**
 * Cap on how many skipped gitignored repos the CLI hint enumerates — a huge
 * gitignored data dir full of clones must never turn the hint scan into a long
 * walk. Enough to make the point; the caller says "+N more" past this.
 */
const UNINDEXED_IGNORED_REPO_HINT_CAP = 100;

/**
 * The INVERSE of the gitignored side of {@link discoverEmbeddedRepoRoots}:
 * nested git repositories under a gitignored directory that the project has NOT
 * opted into via `codegraph.json` `includeIgnored`. These are real repos the
 * default `init`/`index` deliberately skips because `.gitignore` excludes them
 * (#970, #976) — most visibly the "super-repo `.gitignore`s its child repos"
 * layout (#1156), where `init` at the parent correctly indexes ~nothing while
 * `init` inside each child works. The CLI uses this to turn that silent empty
 * index into an actionable hint: it names the skipped repos and offers to opt
 * them in. Paths are `rootDir`-relative and trailing-slashed (valid
 * `includeIgnored` patterns as-is). Returns `[]` for a non-git root (a
 * filesystem walk already descends into nested repos there), skips built-in
 * default-ignored dirs (`node_modules`, …), and is bounded so it never stalls
 * on a giant ignored tree.
 */
export function findUnindexedIgnoredRepos(rootDir: string): string[] {
  try {
    execFileSync('git', ['rev-parse', '--git-dir'], { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
  } catch {
    return [];
  }
  const defaults = defaultsOnlyIgnore();
  const includeIgnored = loadIncludeIgnoredMatcher(rootDir);
  const repos: string[] = [];
  for (const dir of listIgnoredDirs(rootDir)) {
    if (defaults.ignores(dir)) continue; // node_modules etc. — never project code
    if (includeIgnored?.ignores(normalizePath(dir))) continue; // already opted in — nothing to nag about
    for (const repo of findNestedGitRepos(path.join(rootDir, dir), dir)) {
      // Per-repo opt-in check, mirroring findIgnoredEmbeddedRepos: a child
      // pattern (`repos/a/`) doesn't match the parent dir above but DOES
      // cover this repo — it's indexed, so don't nag about it (#1295).
      if (includeIgnored?.ignores(normalizePath(repo))) continue;
      repos.push(repo);
      if (repos.length >= UNINDEXED_IGNORED_REPO_HINT_CAP) return repos;
    }
  }
  return repos;
}

/**
 * Discover embedded repos hidden by `repoDir`'s OWN gitignore rules: for each
 * gitignored directory, search for nested `.git` roots. Returns repo paths
 * relative to `repoDir`, trailing-slashed.
 *
 * OPT-IN ONLY. Walking into a gitignored directory contradicts what every other
 * tool (and CodeGraph's own `git ls-files` foundation) does — `.gitignore`
 * excludes. So this returns `[]` unless the project opted the directory in via
 * `codegraph.json` `includeIgnored`; without that, a gitignored dir — including
 * a huge reference/data dir full of nested clones — is left untouched (#970,
 * #976). When opted in, it restores the super-repo-of-clones behavior (#622,
 * #699). `prefix` is the scan-root-relative path of `repoDir`, so a pattern like
 * `services/` opts that whole subtree in at any recursion depth. Built-in
 * default excludes (`node_modules`, …) are always skipped.
 */
function findIgnoredEmbeddedRepos(repoDir: string, includeIgnored: Ignore | null, prefix: string): string[] {
  if (!includeIgnored) return [];
  const defaults = defaultsOnlyIgnore();
  const repos: string[] = [];
  for (const dir of listIgnoredDirs(repoDir)) {
    if (defaults.ignores(dir)) continue;
    const nested = findNestedGitRepos(path.join(repoDir, dir), dir);
    if (includeIgnored.ignores(normalizePath(prefix + dir))) {
      // The whole ignored dir is opted in — every nested repo under it counts.
      repos.push(...nested);
    } else {
      // A single gitignore rule often covers the PARENT of the opted-in
      // repos: `.gitignore: /repos/` lists `repos/` as ONE ignored entry,
      // while `includeIgnored: ["repos/a/"]` (the CLI hint's own suggested
      // spelling) names the child — which never matches the parent path, so
      // the opt-in silently did nothing (#1295). Match each nested repo
      // root individually so both spellings work. The walk is bounded
      // (depth/entry caps in findNestedGitRepos) and only runs when
      // includeIgnored is configured at all.
      repos.push(...nested.filter((r) => includeIgnored.ignores(normalizePath(prefix + r))));
    }
  }
  return repos;
}

/**
 * `git ls-files -z -s`, expanding submodules where git allows it.
 *
 * `--recurse-submodules` could not be combined with `-s` before git 2.36:
 * `builtin/ls-files.c` listed `show_stage` among the modes that die, and the
 * check is unconditional — it does not look at whether the repo actually has
 * submodules, so every call fails on older git. Ubuntu 22.04 LTS (2.34.1) and
 * Debian 11 (2.30.2) are both below that line.
 *
 * Letting the throw escape cost far more than submodule expansion: it unwound
 * the whole git-visible pass, so `includeIgnored`, gitlink recursion and the
 * `codegraph.json` include allowlist silently stopped applying and files went
 * missing from the index with no error (#1549). Retry without the flag instead
 * — `-s` is the part that matters here, since gitlink detection reads the mode
 * bits, and embedded repos are reached through the gitlink recursion anyway.
 */
function lsFilesStaged(gitOpts: Parameters<typeof execFileSync>[2]): string {
  try {
    return execFileSync('git', ['ls-files', '-z', '-s', '--recurse-submodules'], gitOpts) as unknown as string;
  } catch {
    return execFileSync('git', ['ls-files', '-z', '-s'], gitOpts) as unknown as string;
  }
}

/**
 * Collect git-visible files (tracked + untracked, .gitignore-respected) from the
 * git repository rooted at `repoDir`, adding each to `files` with `prefix`
 * prepended so paths stay relative to the original scan root.
 *
 * Recurses into embedded git repositories — nested repos that are NOT submodules
 * (independent clones living inside the workspace, common in CMake "super-repo"
 * layouts). The parent repo's `git ls-files` cannot see into them: tracked output
 * skips them entirely, and untracked output reports them only as an opaque
 * "subdir/" entry (trailing slash) rather than expanding their files. Each
 * embedded repo is its own git boundary, so we re-run `git ls-files` inside it.
 * (See issue #193.) GITIGNORED embedded repos are invisible even to that; they
 * are discovered separately via `findIgnoredEmbeddedRepos` (#514) but ONLY for
 * directories the project opted in through `codegraph.json` `includeIgnored`
 * (`includeIgnored` here, threaded from the scan root) — by default `.gitignore`
 * is respected and they stay out (#970, #976). Every embedded repo root (however
 * found) is recorded in `embeddedRoots` so callers can exempt its files from the
 * parent's own gitignore rules.
 */
function collectGitFiles(repoDir: string, prefix: string, files: Set<string>, embeddedRoots?: Set<string>, includeIgnored: Ignore | null = null): void {
  const gitOpts = { cwd: repoDir, encoding: 'utf-8' as const, timeout: 30000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'] as ['pipe', 'pipe', 'pipe'], windowsHide: true };

  // Tracked files. --recurse-submodules pulls in files from active submodules,
  // which the index would otherwise represent only as a commit pointer.
  // Without this, monorepos using submodules index 0 files. (See issue #147.)
  // Note: --recurse-submodules only supports -c/--cached and --stage modes — it
  // can't be combined with -o, so untracked files are gathered separately below.
  //
  // We use --stage (-s) rather than -c so each entry carries its file mode. That
  // lets us spot gitlink entries (mode 160000) that --recurse-submodules did NOT
  // expand: a nested repo `git add`ed without a `.gitmodules` entry, or a
  // submodule that isn't active/initialized in this checkout. Such a gitlink
  // falls through every pass — it's tracked, so the untracked `-o` listing below
  // never reports it, and --recurse-submodules only expands ACTIVE submodules —
  // so its source would be silently skipped, leaving only the super-repo's own
  // files indexed. We collect those gitlinks here and recurse into them below.
  // (An active submodule is expanded inline by --recurse-submodules and so never
  // surfaces as a 160000 entry — only the unhandled gitlinks do.) (#1031, #1033)
  //
  // -z gives NUL-separated, unquoted output so non-ASCII (e.g. CJK) paths
  // survive verbatim. Without it git octal-escapes and double-quotes such paths
  // (the core.quotepath default), and the quoted form never matches a real file
  // on disk → those files are silently dropped from the index. (#541) With -s the
  // path follows a TAB after the `<mode> <object> <stage>` prefix.
  const gitlinkRels: string[] = [];
  const tracked = lsFilesStaged(gitOpts);
  for (const entry of tracked.split('\0')) {
    if (!entry) continue;
    const tab = entry.indexOf('\t');
    if (tab === -1) continue; // --stage always emits "<mode> <object> <stage>\t<path>"
    const rel = entry.slice(tab + 1);
    if (entry.slice(0, 6) === '160000') {
      gitlinkRels.push(rel); // an unexpanded gitlink — recursed into below, not a source file itself
      continue;
    }
    files.add(normalizePath(prefix + rel));
  }

  // Untracked files (submodules manage their own untracked state). Embedded git
  // repos surface here as a single "subdir/" entry that git refuses to descend
  // into — recurse into those as their own repos so their source gets indexed.
  const untracked = execFileSync('git', ['ls-files', '-z', '-o', '--exclude-standard'], gitOpts);
  for (const rel of untracked.split('\0')) {
    if (!rel) continue;
    if (rel.endsWith('/')) {
      // git only emits a trailing-slash directory entry for an embedded repo.
      // Guard with a .git check anyway, and skip anything else exactly as git
      // itself skips it (we never descend into a non-repo opaque dir). Never
      // descend into default-ignored locations — an embedded repo inside
      // node_modules is an npm git-dependency, not project code.
      const childDir = path.join(repoDir, rel);
      // A git worktree surfaces here as an opaque untracked dir too — skip it,
      // it's a duplicate working view of an already-indexed repo (#848).
      if (classifyGitDir(childDir) === 'embedded' && !defaultsOnlyIgnore().ignores(rel)) {
        embeddedRoots?.add(normalizePath(prefix + rel));
        collectGitFiles(childDir, prefix + rel, files, embeddedRoots, includeIgnored);
      }
      continue;
    }
    files.add(normalizePath(prefix + rel));
  }

  // Gitlink entries (mode 160000) that --recurse-submodules left unexpanded —
  // an embedded repo `git add`ed without `.gitmodules`, or a submodule not
  // active/initialized in this checkout. When such a gitlink has a real working
  // tree on disk it is distinct first-party code we must index as its own
  // embedded repo: the tracked pass skipped its contents and the untracked pass
  // never sees it (it's tracked, not "other"). A gitlink with no checkout on disk
  // (an uninitialized submodule — empty dir, no `.git`) has nothing to index and
  // is left alone, as is a submodule worktree (a duplicate view, #945). (#1031, #1033)
  if (gitlinkRels.length > 0) {
    const defaults = defaultsOnlyIgnore();
    const repoIgnore = buildDefaultIgnore(repoDir);
    for (const rel of gitlinkRels) {
      const relDir = rel.endsWith('/') ? rel : rel + '/';
      // A gitlink under a gitignored path is respected (not indexed) unless the
      // project opted it in via `includeIgnored` — keep tracked gitlinks under
      // the same scope rule as the untracked-ignored kind below (#1065).
      if (gitlinkEmbeddedRepoSkipped(relDir, prefix, defaults, repoIgnore, includeIgnored)) continue;
      const childDir = path.join(repoDir, rel);
      // 'embedded' = a real .git checkout on disk; 'worktree' and 'none' are skipped.
      if (classifyGitDir(childDir) !== 'embedded') continue;
      embeddedRoots?.add(normalizePath(prefix + relDir));
      collectGitFiles(childDir, prefix + relDir, files, embeddedRoots, includeIgnored);
    }
  }

  // Embedded repos hidden by THIS repo's ignore rules (`/packages/` in a
  // super-repo .gitignore) never appear in any listing above. By default they
  // stay hidden — `.gitignore` is respected (#970, #976). They are recursed into
  // only when the project opted the directory in via `codegraph.json`
  // `includeIgnored` (#622, #699), which `findIgnoredEmbeddedRepos` enforces.
  for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
    embeddedRoots?.add(normalizePath(prefix + rel));
    collectGitFiles(path.join(repoDir, rel), prefix + rel, files, embeddedRoots, includeIgnored);
  }
}

/**
 * Get all files visible to git (tracked + untracked but not ignored).
 * Respects .gitignore at all levels (root, subdirectories) and descends into
 * embedded (nested, non-submodule) git repos. Returns null on failure
 * (non-git project) so callers can fall back to a filesystem walk.
 */
function getGitVisibleFiles(rootDir: string): Set<string> | null {
  try {
    // Check if the project directory is gitignored by a parent repo.
    // When rootDir lives inside a parent git repo that ignores it,
    // `git ls-files` returns nothing — fall back to filesystem walk.
    const gitRoot = execFileSync(
      'git',
      ['rev-parse', '--show-toplevel'],
      { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }
    ).trim();

    if (path.resolve(gitRoot) !== path.resolve(rootDir)) {
      try {
        // git check-ignore exits 0 if the path IS ignored, 1 if not
        execFileSync(
          'git',
          ['check-ignore', '-q', path.resolve(rootDir)],
          { cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }
        );
        // Directory is gitignored by parent repo — fall back to filesystem walk
        logDebug('project root is gitignored by a parent repo — falling back to filesystem walk', {
          rootDir,
          gitRoot,
        });
        return null;
      } catch {
        // Not ignored — safe to use git ls-files
      }
    }

    const files = new Set<string>();
    const embeddedRoots = new Set<string>();
    collectGitFiles(rootDir, '', files, embeddedRoots, loadIncludeIgnoredMatcher(rootDir));
    // Apply built-in default ignores uniformly — to tracked files too, since
    // committing a dependency/build dir doesn't make it project code. A
    // `.gitignore` negation (e.g. `!vendor/`) is the explicit opt-in. (issue #407)
    // Files inside an EMBEDDED repo are matched against that repo's own rules,
    // not the parent's: the parent's .gitignore hides the child repo from git,
    // not from the index. (#514)
    const ig = buildScopeIgnore(rootDir, embeddedRoots);
    const visible = new Set([...files].filter((f) => !ig.ignores(f)));
    // Force-include first-party source the project whitelisted in
    // `codegraph.json` `include`. These are gitignored, so `git ls-files` never
    // listed them above — discover them directly off disk and add them. (The
    // common SVN+Git dual-VCS case: source committed to SVN, gitignored out of
    // Git, but still wanted in the graph.)
    for (const f of collectIncludedFilesForRoot(rootDir)) visible.add(f);
    return visible;
  } catch (error) {
    // Any failure here (git missing, a `git rev-parse`/`ls-files` timeout or
    // buffer overrun under load, an unreadable repo, unsupported flag combo on
    // older git, etc.) silently sent every caller to `scanDirectoryWalk` with
    // zero signal that the fast git-delegated path was skipped — making reports
    // like #1567 (nested-`.gitignore`-excluded `node_modules` walked into)
    // hard to triage, since both ignore implementations look correct in
    // isolation but there was no way to tell which one ran. Log it under the
    // existing CODEGRAPH_DEBUG gate so a future report can confirm or rule out
    // the fallback in one step.
    logDebug('git-based file listing unavailable — falling back to filesystem walk', {
      rootDir,
      error: error instanceof Error ? error.message : String(error),
    });
    return null;
  }
}

/**
 * Result of git-based change detection.
 * Returns null when git is unavailable (non-git project or command failure),
 * signaling the caller to fall back to full filesystem scan.
 */
interface GitChanges {
  modified: string[];  // M, MM, AM — files to re-hash + re-index
  added: string[];     // ?? — new untracked files to index
  deleted: string[];   // D — files to remove from DB
}

/**
 * Use `git status` to detect changed files instead of scanning every file.
 * Returns null on failure so callers fall back to full scan.
 *
 * Recurses into embedded repos — the untracked kind (#193: the parent's status
 * collapses them to an opaque `?? subdir/` entry) always, and the gitignored
 * kind (#514: they never appear in the parent's status at all) only for
 * directories opted in via `codegraph.json` `includeIgnored` (#622, #699) —
 * running `git status` inside each, so changes in a multi-repo workspace sync
 * without a full rescan. By default a gitignored dir is left alone, matching the
 * full-index scan (#970, #976). Deleting an ENTIRE embedded repo dir is the one
 * case this cannot see (the child status that would report the deletions is gone
 * with it); a full `codegraph index` reconciles that.
 */
export function getGitChangedFiles(rootDir: string, sinceCommit?: string | null): GitChanges | null {
  try {
    // `git status` only ever describes the WORKING TREE, so a change that has
    // been committed leaves no entry and never enters the candidate set — the
    // hash comparison in getChangedFiles is correct but is never reached for
    // it, and `pendingChanges` reads 0 while the index is genuinely behind
    // (#1829). `sinceCommit` — the commit the index was last brought up to
    // date at — adds the other half: what has been committed since. Callers
    // that hold no such stamp still get exactly what they always did, the
    // working-tree changes.
    const changes: GitChanges = { modified: [], added: [], deleted: [] };
    // Custom extension → language overrides from the project's codegraph.json,
    // so change detection sees the same custom-extension files the full index does.
    const overrides = loadExtensionOverrides(rootDir);
    collectGitStatus(rootDir, '', changes, overrides, loadIncludeIgnoredMatcher(rootDir), loadExcludeMatcher(rootDir), sinceCommit ?? undefined);
    return changes;
  } catch {
    return null;
  }
}

/**
 * Metadata key: the commit the index was last brought up to date at. Written by
 * a full index AND by every successful sync — unlike the extraction stamp, which
 * a sync must not advance because it only touches a subset of files. This one is
 * about the tree; failed file paths remain explicit retry candidates. (#1829)
 */
export const INDEXED_AT_COMMIT_KEY = 'indexed_at_commit';

/** HEAD's commit sha, or null in a non-git repo or one with no commits yet. */
export function getGitHeadSha(rootDir: string): string | null {
  try {
    return execFileSync('git', ['rev-parse', 'HEAD'], {
      cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true,
    }).trim() || null;
  } catch {
    return null;
  }
}

/**
 * NUL-delimited status/path pairs for every path committed
 * between `sinceCommit` and HEAD. Empty when the stamp IS HEAD, which is the
 * common case — one cheap git call on the hot path.
 */
function gitCommittedChangesSince(repoDir: string, sinceCommit: string): string[] {
  // NUL framing preserves Unicode, quotes, tabs and newlines in Git paths.
  // Let command failures reach getGitChangedFiles: [] would falsely mean clean.
  const out = execFileSync('git', ['diff', '--relative', '--name-status', '--no-renames', '-z', sinceCommit, 'HEAD', '--', '.'], {
    cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true,
  });
  return out.split('\0');
}

/**
 * Can an INDEX trust the git fast path, given the commit it was built at?
 *
 * False means "fall back to the full scan" — the expensive path that compares
 * every file on disk against the DB, and the only correct read when git cannot
 * say what happened between the stamp and now:
 *
 *  - stamp present but unknown to this repo (rebase, gc, shallow clone, a stamp
 *    from a different checkout) — history moved under the index.
 *  - stamp absent while the repo HAS commits — an index built before stamping
 *    existed. One full scan; the next sync stamps it and the fast path returns.
 *
 * A repo with NO commits keeps the fast path with or without a stamp: every
 * file is untracked, so `git status` already sees all of them. Callers with no
 * index behind them (the exported `getGitChangedFiles`) never ask this — a
 * working-tree diff is the whole of what they wanted. (#1829)
 */
export function canTrustGitFastPath(rootDir: string, sinceCommit?: string | null): boolean {
  const head = getGitHeadSha(rootDir);
  if (head == null) return true; // no commits (or not a git repo — caller handles that)
  if (!sinceCommit) return false;
  if (sinceCommit === head) return true;
  try {
    execFileSync('git', ['cat-file', '-e', `${sinceCommit}^{commit}`], {
      cwd: rootDir, encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true,
    });
    return true;
  } catch {
    return false;
  }
}

function collectGitStatus(repoDir: string, prefix: string, out: GitChanges, overrides?: Record<string, Language>, includeIgnored: Ignore | null = null, exclude: Ignore | null = null, sinceCommit?: string): void {
  const output = execFileSync(
    'git',
    // `-uall` lists individual untracked files instead of collapsing an
    // entirely-untracked directory into one `?? dir/` entry, which would
    // otherwise be dropped here (only embedded git repos are recursed into
    // below). Nested untracked git repos still collapse to `?? repo/` even
    // with `-uall` — git never crosses a repo boundary — so the recursion
    // still handles them. (#1213)
    ['status', '--porcelain', '--no-renames', '-z', '-uall'],
    { cwd: repoDir, encoding: 'utf-8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }
  );

  // This repo's own ignore rules — built-in defaults (#407) plus its .gitignore.
  // Change detection must exclude the SAME files the full index does, but git
  // status hides neither: it ignores nothing for *tracked* paths, and the
  // built-in defaults aren't gitignore at all. Without this filter a committed
  // vendor/ dir, or a tracked file under a .gitignored dir, surfaces here as a
  // change — so `codegraph status` (which reads getChangedFiles) reports a
  // pending edit the full index never tracks and `sync` never clears. Matching
  // repo-relative `rel` at each recursion level mirrors getGitVisibleFiles'
  // ScopeIgnore: every embedded repo is judged by ITS OWN rules, never the
  // parent's. (#766)
  const ig = buildDefaultIgnore(repoDir);

  // One classifier for both candidate sources below, so a committed change is
  // filtered by exactly the rules a working-tree change is (#766, #999, #1829).
  const classify = (statusCode: string, rel: string): void => {
    const filePath = normalizePath(prefix + rel);
    if (!isSourceFile(filePath, overrides)) return;

    if (statusCode.includes('D')) {
      // Deletions stay unfiltered: getChangedFiles acts on one only when the
      // path is already tracked in the DB, where removal is always correct — and
      // that lets a newly-excluded dir's stale rows clean themselves up. (#766)
      out.deleted.push(filePath);
      return;
    }

    // Added (`??`) / modified files inside an excluded dir must not enter the
    // index — match against the repo-relative path, same as the full scan. (#766)
    if (ig.ignores(rel)) return;
    // User `codegraph.json` `exclude` (#999) is project-root-relative, so it's
    // matched against the full path — sync must not re-add a tracked file the
    // full index now keeps out. Deletions above stay unfiltered so a file that
    // WAS indexed before an exclude was added still cleans itself out.
    if (exclude && exclude.ignores(filePath)) return;

    if (statusCode === '??') {
      out.added.push(filePath);
    } else {
      // M, MM, AM, A (staged), etc. — treat as modified. getChangedFiles
      // re-decides added-vs-modified from the DB, so a committed `A` that the
      // index never saw still lands in `added`.
      out.modified.push(filePath);
    }
  };

  const untrackedDirs: string[] = [];
  for (const line of output.split('\0')) {
    if (line.length < 4) continue; // Minimum: "XY file"

    const statusCode = line.substring(0, 2);
    const rel = normalizePath(line.substring(3));

    // Untracked directory entries (trailing slash) may hide an embedded repo —
    // collect for the recursion below instead of treating as a file.
    if (statusCode === '??' && rel.endsWith('/')) {
      untrackedDirs.push(rel);
      continue;
    }

    classify(statusCode, rel);
  }

  // Committed but unindexed: everything between the commit this index was last
  // brought up to date at and HEAD. `git status` cannot see these — committing
  // is precisely what removes a file from its output — so without this pass a
  // `git commit` makes a real pending change read as zero (#1829). The stamp
  // belongs to the ROOT repo, so the embedded-repo recursion below passes none.
  if (sinceCommit) {
    const fields = gitCommittedChangesSince(repoDir, sinceCommit);
    for (let i = 0; i + 1 < fields.length; i += 2) {
      classify(`${fields[i]!.charAt(0)} `, normalizePath(fields[i + 1]!));
    }
  }

  // Recurse embedded repos found under untracked dirs (at the dir itself or
  // nested deeper). Gitignored dirs are walked only for the directories the
  // project opted in via `includeIgnored`; by default `.gitignore` is respected
  // and they are left alone (#970, #976), mirroring the full-index scan.
  for (const rel of untrackedDirs) {
    for (const repoRel of findNestedGitRepos(path.join(repoDir, rel), rel)) {
      collectGitStatus(path.join(repoDir, repoRel), prefix + repoRel, out, overrides, includeIgnored, exclude);
    }
  }
  for (const rel of findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix)) {
    collectGitStatus(path.join(repoDir, rel), prefix + rel, out, overrides, includeIgnored, exclude);
  }
}

/**
 * Recursively scan a directory for source files.
 *
 * In git repos, uses `git ls-files` (inherently respects .gitignore at all
 * levels), then keeps files with a supported source extension. For non-git
 * projects, falls back to a filesystem walk that parses .gitignore itself.
 */
export function scanDirectory(
  rootDir: string,
  onProgress?: (current: number, file: string) => void
): string[] {
  // Custom extension → language overrides from the project's codegraph.json.
  const overrides = loadExtensionOverrides(rootDir);

  // Fast path: use git to get all visible files (respects .gitignore everywhere)
  const gitFiles = getGitVisibleFiles(rootDir);
  if (gitFiles) {
    const files: string[] = [];
    let count = 0;
    for (const filePath of gitFiles) {
      if (isSourceFile(filePath, overrides)) {
        files.push(filePath);
        count++;
        onProgress?.(count, filePath);
      }
    }
    return files;
  }

  // Fallback: walk filesystem for non-git projects
  return scanDirectoryWalk(rootDir, onProgress);
}

/**
 * Async variant of scanDirectory that yields to the event loop periodically,
 * allowing worker threads to receive and render progress messages.
 */
/**
 * What a scan saw but could not index, tallied by extension.
 *
 * Filled during the walk the scan already performs — a project of unsupported
 * files is otherwise indistinguishable from an empty one, because unsupported
 * extensions are filtered out at discovery and never counted anywhere (#1502).
 */
export interface ScanSkipStats {
  /** Lowercased extension (with dot) → how many files carried it. */
  unsupportedByExtension: Map<string, number>;
}

/** Record one file the scan declined to index. */
function tallySkip(stats: ScanSkipStats | undefined, rel: string): void {
  if (!stats) return;
  const ext = path.extname(rel).toLowerCase();
  if (!ext) return;
  stats.unsupportedByExtension.set(ext, (stats.unsupportedByExtension.get(ext) ?? 0) + 1);
}

export async function scanDirectoryAsync(
  rootDir: string,
  onProgress?: (current: number, file: string) => void,
  stats?: ScanSkipStats
): Promise<string[]> {
  // Custom extension → language overrides from the project's codegraph.json.
  const overrides = loadExtensionOverrides(rootDir);

  const gitFiles = getGitVisibleFiles(rootDir);
  if (gitFiles) {
    const files: string[] = [];
    let count = 0;
    for (const filePath of gitFiles) {
      if (isSourceFile(filePath, overrides)) {
        files.push(filePath);
        count++;
        onProgress?.(count, filePath);
        // Yield every 100 files so worker threads can render progress
        if (count % 100 === 0) {
          await new Promise<void>(r => setImmediate(r));
        }
      } else {
        tallySkip(stats, filePath);
      }
    }
    return files;
  }

  return scanDirectoryWalk(rootDir, onProgress, stats);
}

/**
 * Filesystem walk fallback for non-git projects.
 */
function scanDirectoryWalk(
  rootDir: string,
  onProgress?: (current: number, file: string) => void,
  stats?: ScanSkipStats
): string[] {
  const files: string[] = [];
  let count = 0;
  const visitedDirs = new Set<string>();
  // Custom extension → language overrides from the project's codegraph.json.
  const overrides = loadExtensionOverrides(rootDir);

  // A .gitignore matcher scoped to the directory that declared it. Patterns in
  // a nested .gitignore are relative to that directory, so we keep the dir
  // alongside the matcher and test paths relative to it — mirroring how git
  // applies .gitignore files at every level.
  interface ScopedIgnore {
    dir: string;
    ig: Ignore;
  }

  const loadIgnore = (dir: string): ScopedIgnore | null => {
    const giPath = path.join(dir, '.gitignore');
    if (!fs.existsSync(giPath)) return null;
    // readGitignorePatterns is defensive: a non-UTF-8 (DLP-encrypted) or
    // uncompilable .gitignore is skipped/filtered with a warning, never thrown
    // (issue #682) — so the per-file `.ignores()` calls below can't crash.
    const patterns = readGitignorePatterns(giPath);
    return patterns ? { dir, ig: ignore().add(patterns) } : null;
  };

  const isIgnored = (fullPath: string, isDir: boolean, matchers: ScopedIgnore[]): boolean => {
    for (const { dir, ig } of matchers) {
      let rel = normalizePath(path.relative(dir, fullPath));
      if (!rel || rel.startsWith('..')) continue; // not under this matcher's dir
      if (isDir) rel += '/'; // dir-only rules (e.g. `build/`) only match with the slash
      if (ig.ignores(rel)) return true;
    }
    return false;
  };

  function walk(dir: string, matchers: ScopedIgnore[]): void {
    let realDir: string;
    try {
      realDir = fs.realpathSync(dir);
    } catch {
      logDebug('Skipping unresolvable directory', { dir });
      return;
    }

    if (visitedDirs.has(realDir)) {
      logDebug('Skipping already-visited directory (symlink cycle)', { dir, realDir });
      return;
    }
    visitedDirs.add(realDir);

    // This directory's own .gitignore (if present) applies to everything below it.
    // The root's .gitignore is already merged into the seeded base matcher (so a
    // negation there can override a built-in default), so skip it here.
    const own = dir === rootDir ? null : loadIgnore(dir);
    const active = own ? [...matchers, own] : matchers;

    let entries: fs.Dirent[];
    try {
      entries = fs.readdirSync(dir, { withFileTypes: true });
    } catch (error) {
      logDebug('Skipping unreadable directory', { dir, error: String(error) });
      return;
    }

    for (const entry of entries) {
      // Never descend into git internals or any CodeGraph data directory
      // (the active one or a sibling another environment created — #636).
      if (entry.name === '.git' || isCodeGraphDataDir(entry.name)) continue;

      const fullPath = path.join(dir, entry.name);
      const relativePath = normalizePath(path.relative(rootDir, fullPath));

      if (entry.isSymbolicLink()) {
        try {
          const realTarget = fs.realpathSync(fullPath);
          const stat = fs.statSync(realTarget);
          if (stat.isDirectory()) {
            if (!isIgnored(fullPath, true, active)) {
              walk(fullPath, active);
            }
          } else if (stat.isFile()) {
            if (!isIgnored(fullPath, false, active)) {
              if (isSourceFile(relativePath, overrides)) {
                files.push(relativePath);
                count++;
                onProgress?.(count, relativePath);
              } else {
                tallySkip(stats, relativePath);
              }
            }
          }
        } catch {
          logDebug('Skipping broken symlink', { path: fullPath });
        }
        continue;
      }

      if (entry.isDirectory()) {
        if (!isIgnored(fullPath, true, active)) {
          walk(fullPath, active);
        }
      } else if (entry.isFile()) {
        if (!isIgnored(fullPath, false, active)) {
          if (isSourceFile(relativePath, overrides)) {
            files.push(relativePath);
            count++;
            onProgress?.(count, relativePath);
          } else {
            tallySkip(stats, relativePath);
          }
        }
      }
    }
  }

  // Seed a base matcher with the built-in default ignores (merged with the root
  // .gitignore so a negation can override). Nested .gitignores still layer per-dir.
  const baseMatchers: ScopedIgnore[] = [{ dir: rootDir, ig: buildDefaultIgnore(rootDir) }];
  // Project `codegraph.json` `exclude` patterns (#999), rooted at the project so
  // `isIgnored` matches them against root-relative paths — same coverage the
  // git path gets via ScopeIgnore, for non-git projects.
  const exclude = loadExcludeMatcher(rootDir);
  if (exclude) baseMatchers.push({ dir: rootDir, ig: exclude });
  walk(rootDir, baseMatchers);

  // Force-include first-party source whitelisted in `codegraph.json` `include`
  // — the walk above honours `.gitignore`, so anything gitignored was dropped;
  // add it back here (deduped). Mirrors the git path's union.
  const included = collectIncludedFilesForRoot(rootDir);
  if (included.size > 0) {
    const seen = new Set(files);
    for (const f of included) {
      if (!seen.has(f)) {
        files.push(f);
        seen.add(f);
      }
    }
  }
  return files;
}

/**
 * Resurrect a resolution edge that is about to be dropped (its target symbol
 * was removed, renamed, or its whole file deleted) as the ORIGINAL unresolved
 * reference that created it, read from the refName/refKind stamp
 * `createEdges` writes into edge metadata. Inserted as status='pending', the
 * ref is consumed by the same sync's resolution sweep: it rebinds to an
 * alternative definition if one exists, or parks as status='failed' where the
 * #1240 retry finds it if the symbol later reappears.
 *
 * Returns null — drop silently, the pre-#1240 behavior — for edges without a
 * refName stamp (created before the stamp existed, or synthesized): rebuilding
 * a ref from the target's plain node name would strip the receiver/qualifier
 * context the original text carried (`h.greet` → `greet`) and could rebind
 * somewhere a full re-index never would. Silent beats wrong.
 */
function resurrectRefFromDroppedEdge(
  e: Edge & { sourceFilePath: string; sourceLanguage: Language }
): UnresolvedReference | null {
  const refName = e.metadata?.refName;
  if (typeof refName !== 'string' || refName.length === 0) return null;
  const refKind = typeof e.metadata?.refKind === 'string' ? (e.metadata.refKind as ReferenceKind) : e.kind;
  return {
    fromNodeId: e.source,
    referenceName: refName,
    referenceKind: refKind,
    line: e.line ?? 0,
    column: e.column ?? 0,
    filePath: e.sourceFilePath,
    language: e.sourceLanguage,
  };
}

/**
 * Extraction orchestrator
 */
export class ExtractionOrchestrator {
  private rootDir: string;
  private queries: QueryBuilder;
  /**
   * Names of frameworks detected for this project, populated by indexAll().
   * Passed to extractFromSource so framework-specific extractors (route nodes,
   * middleware, etc.) run after the tree-sitter pass. Cleared if detection
   * hasn't run yet so single-file re-index paths can detect on the spot.
   */
  private detectedFrameworkNames: string[] | null = null;
  /**
   * Scope matcher for SCOPED syncs, memoized on the mtimes of the two root
   * files it is derived from (`codegraph.json`, `.gitignore`). See
   * {@link scopedSyncMatcher}.
   */
  private scopedMatcher: { key: string; matcher: ScopeIgnore } | null = null;

  constructor(rootDir: string, queries: QueryBuilder) {
    this.rootDir = rootDir;
    this.queries = queries;
  }

  /**
   * The scope matcher a scoped sync applies to the paths it was handed — the
   * same `buildScopeIgnore` the full scan uses, so an explicitly-passed path
   * that is OUT of scope (a user `exclude` in `codegraph.json`, a `.gitignore`
   * rule, a built-in default) is treated exactly as the full walk would treat
   * it: absent, hence removed if tracked, never parsed (#1590).
   *
   * Memoized on the root config + root `.gitignore` mtimes: building the
   * matcher runs embedded-repo discovery (`git ls-files`), which would defeat
   * the scoped path's whole point (skipping O(repo) work) if paid per sync.
   * Two `stat`s per sync while nothing changed. An embedded repo created
   * between config edits joins the scoped matcher on the next full sync, the
   * same lifecycle the watcher's own matcher already has.
   */
  private scopedSyncMatcher(): ScopeIgnore {
    // Bust when any root-level exclude source the matcher reads may have
    // changed. Nested `.gitignore` edits force a full watcher sync, which
    // clears this cache (see the full-reconcile branch in sync()).
    const gitDir = resolveGitDir(this.rootDir);
    const key = [
      PROJECT_CONFIG_FILENAME,
      '.gitignore',
      gitDir ? path.join(gitDir, 'info', 'exclude') : '',
    ]
      .map((name) => {
        if (!name) return '-';
        try {
          return String(fs.statSync(path.isAbsolute(name) ? name : path.join(this.rootDir, name)).mtimeMs);
        } catch {
          return '-';
        }
      })
      .join('|');
    if (this.scopedMatcher && this.scopedMatcher.key === key) return this.scopedMatcher.matcher;
    const matcher = buildScopeIgnore(this.rootDir);
    this.scopedMatcher = { key, matcher };
    return matcher;
  }

  /**
   * Build a filesystem-backed ResolutionContext sufficient for framework
   * detection. Graph-query methods (getNodesByName etc.) return empty because
   * the DB hasn't been populated yet, but detect() only uses readFile,
   * fileExists, and getAllFiles, so that's fine.
   */
  private buildDetectionContext(files: string[]): ResolutionContext {
    const rootDir = this.rootDir;
    return {
      getNodesInFile: () => [],
      getNodesByName: () => [],
      getNodesByQualifiedName: () => [],
      getNodesByKind: () => [],
      getNodesByLowerName: () => [],
      getImportMappings: () => [],
      getAllFiles: () => files,
      getProjectRoot: () => rootDir,
      fileExists: (relativePath: string) => {
        const full = validatePathWithinRoot(rootDir, relativePath);
        if (!full) return false;
        try {
          return fs.existsSync(full);
        } catch {
          return false;
        }
      },
      readFile: (relativePath: string) => {
        const full = validatePathWithinRoot(rootDir, relativePath);
        if (!full) return null;
        try {
          return fs.readFileSync(full, 'utf-8');
        } catch {
          return null;
        }
      },
      // Monorepo support — needed by framework detect()s that probe
      // subpackage manifests (e.g. fabric-view looking at
      // packages/<sub>/package.json when the root manifest is just a
      // workspace declaration). Matches the resolver-context shape.
      listDirectories: (relativePath: string) => {
        const target =
          relativePath === '.' || relativePath === ''
            ? rootDir
            : path.join(rootDir, relativePath);
        try {
          return fs
            .readdirSync(target, { withFileTypes: true })
            .filter((entry) => entry.isDirectory())
            .map((entry) => entry.name);
        } catch {
          return [];
        }
      },
    };
  }

  /**
   * Detect frameworks on demand using the current scanned files (or a fresh
   * scan if none are provided). Cached on the orchestrator so repeat calls
   * inside a single run don't re-scan.
   */
  private ensureDetectedFrameworks(files?: string[]): string[] {
    if (this.detectedFrameworkNames !== null) return this.detectedFrameworkNames;
    const fileList = files ?? scanDirectory(this.rootDir);
    const context = this.buildDetectionContext(fileList);
    this.detectedFrameworkNames = detectFrameworks(context).map((r) => r.name);
    return this.detectedFrameworkNames;
  }

  /**
   * Index all files in the project
   */
  async indexAll(
    onProgress?: (progress: IndexProgress) => void,
    signal?: AbortSignal,
    verbose?: boolean,
    // Writer-side backstop for deferred WAL checkpointing (#1231): returns
    // null in the normal case, or a promise to await (at this safe,
    // between-transactions boundary) when the WAL has outrun the off-thread
    // checkpointer past its hard cap. See db/wal-valve.ts.
    walBackpressure?: () => Promise<void> | null,
    // Fresh-DB store offload (perf): when set, per-file store bundles are
    // applied by a dedicated writer thread instead of the main thread. Only
    // passed for a COMPLETELY fresh database, where the main thread performs
    // no reads/writes during the parse loop, so one writer applying bundles
    // in file order preserves the #1015 determinism exactly.
    storeWriterOpts?: { dbPath: string; fastInit: boolean } | null
  ): Promise<IndexResult> {
    const tGrammar = Date.now();
    await initGrammars();
    if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] grammar-init: ${Date.now() - tGrammar}ms`);
    const startTime = Date.now();
    const errors: ExtractionError[] = [];
    let filesIndexed = 0;
    let filesSkipped = 0;
    let filesErrored = 0;
    let totalNodes = 0;
    let totalEdges = 0;

    // Custom extension → language overrides from the project's codegraph.json.
    // Threaded into language detection so custom-extension files load the right
    // grammar and store under the mapped language.
    const overrides = loadExtensionOverrides(this.rootDir);

    const log = verbose
      ? (msg: string) => { console.log(`[worker] ${msg}`); }
      : (_msg: string) => {};

    // Phase 1: Scan for files
    onProgress?.({
      phase: 'scanning',
      current: 0,
      total: 0,
    });

    // Phase attribution to stderr (same opt-in as the synthesis timings):
    // early-run 5-10s single stalls were observed on 95k-file repos but never
    // attributed — these labels settle scan vs framework-detect vs grammars.
    const tScan = Date.now();
    const skipStats: ScanSkipStats = { unsupportedByExtension: new Map() };
    const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
      onProgress?.({
        phase: 'scanning',
        current,
        total: 0,
        currentFile: file,
      });
    }, skipStats);
    if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`);
    /** Only meaningful when nothing was indexable — see IndexResult (#1502). */
    const skipSummary = (): Pick<IndexResult, 'filesSkippedUnsupported' | 'topUnsupportedExtensions'> => {
      let total = 0;
      for (const n of skipStats.unsupportedByExtension.values()) total += n;
      if (total === 0) return {};
      const top = [...skipStats.unsupportedByExtension.entries()]
        .map(([ext, count]) => ({ ext, count }))
        .sort((a, b) => b.count - a.count || a.ext.localeCompare(b.ext))
        .slice(0, 5);
      return { filesSkippedUnsupported: total, topUnsupportedExtensions: top };
    };


    // A re-index over an existing DB skips unchanged-hash files at the store,
    // which would preserve wiped zero-node rows (#1541) — drop them first so
    // this run stores their files fresh. No-op on a fresh DB.
    this.healZeroNodeRows();

    // Detect frameworks once per indexAll run using the scanned file list.
    // Names are passed to each parse call so framework-specific extractors
    // (route nodes, middleware, etc.) run after the tree-sitter pass.
    // Framework detection is reset each run so adding e.g. requirements.txt
    // between runs is picked up without restarting the process.
    this.detectedFrameworkNames = null;
    const tFw = Date.now();
    const frameworkNames = this.ensureDetectedFrameworks(files);
    if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] framework-detect: ${Date.now() - tFw}ms`);

    if (signal?.aborted) {
      return {
        success: false,
        filesIndexed: 0,
        filesSkipped: 0,
        filesErrored: 0,
        nodesCreated: 0,
        edgesCreated: 0,
        errors: [{ message: 'Aborted', severity: 'error' }],
        durationMs: Date.now() - startTime,
      };
    }

    // Phase 2: Parse files in a worker thread (keeps main thread unblocked for UI)
    const total = files.length;
    let processed = 0;

    // Emit parsing phase immediately so the progress bar appears during worker setup.
    // The yield lets the shimmer worker flush the phase transition to stdout before
    // the main thread starts synchronous grammar detection work.
    onProgress?.({
      phase: 'parsing',
      current: 0,
      total,
    });
    await new Promise(resolve => setImmediate(resolve));

    // Detect needed languages and load grammars in the parse worker
    const neededLanguages = preloadLanguagesForFiles(files, overrides);

    // Parse files on a pool of worker threads (keeps the main thread free for UI
    // and uses every core). Falls back to in-process parsing when the compiled
    // worker is unavailable (e.g. running from source in tests).
    const parseWorkerPath = path.join(__dirname, 'parse-worker.js');
    const useWorker = fs.existsSync(parseWorkerPath);

    let pool: ParseWorkerPool | null = null;
    if (useWorker) {
      // CODEGRAPH_PARSE_WORKERS: explicit worker count; 1 = the old single-worker
      // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8),
      // with cores from availableParallelism — cpuset/affinity-honest, where
      // os.cpus() enumerates the host's CPUs and spawned 8 wasm workers (and
      // their grammar heaps) inside a 2-CPU container for zero extra
      // throughput (§7a.1). Floored so a 2-core box still gets 2 workers:
      // parse is worker-side CPU, and 1 worker measured 34% slower than the
      // old oversubscribed pool on the kernel-scale 2-cpuset envelope
      // (493s vs 369s) — main + store-worker don't fill the second core.
      const poolSize = resolveParsePoolSize(process.env.CODEGRAPH_PARSE_WORKERS, Math.max(3, os.availableParallelism()));
      // Read each needed grammar's WASM ONCE here and hand the bytes to every
      // worker, so spawns/respawns load grammars from memory instead of
      // re-reading them from disk (#1231: on an HDD, respawn re-reads amplify
      // the very I/O contention that caused the respawn).
      const grammarBuffers = await readGrammarWasmBytes(neededLanguages);
      pool = new ParseWorkerPool({
        languages: neededLanguages,
        size: poolSize,
        workerScriptPath: parseWorkerPath,
        recycleInterval: WORKER_RECYCLE_INTERVAL,
        parseTimeoutMs: PARSE_TIMEOUT_MS,
        log,
        grammarBuffers,
      });
      log(`Parse worker pool: ${poolSize} worker(s)`);
      // Bulk index: every core will be needed — spawn the whole pool now so
      // worker boot overlaps the first read batches instead of trickling in
      // behind queue-pressure growth.
      pool.prewarm();
    } else {
      // In-process fallback: load grammars locally and parse on the main thread.
      await loadGrammarsForLanguages(neededLanguages);
    }

    // Dedicated store writer thread (fresh DB only — see the parameter doc).
    // Same availability rule as the parse pool: needs the compiled worker
    // (absent when running from source in tests → main-thread fallback).
    const storeWorkerPath = path.join(__dirname, 'store-worker.js');
    let storeWriter: StoreWriter | null = null;
    if (
      storeWriterOpts &&
      process.env.CODEGRAPH_NO_STORE_WORKER !== '1' &&
      fs.existsSync(storeWorkerPath)
    ) {
      // Deliberately NOT awaiting ready(): worker_threads delivers messages in
      // order, so bundles posted while the worker is still booting queue
      // behind 'open'. A boot failure surfaces at the first drain() — same
      // propagation point as a store error.
      storeWriter = new StoreWriter(storeWorkerPath, storeWriterOpts.dbPath, storeWriterOpts.fastInit);
      log('Store writer thread active');
    }
    /** Queue-depth bound for un-acked bundles (bundles hold whole node/edge arrays). */
    const STORE_WRITER_WINDOW = 64;

    /**
     * Parse one file: on the pool when available (the promise REJECTS on a worker
     * crash/timeout — the caller records it and the retry pass re-attempts), or
     * in-process synchronously as the no-worker fallback. The language is resolved
     * here on the main thread, where the codegraph.json overrides are loaded.
     */
    const parseFile = (filePath: string, content: string): Promise<ExtractionResult> => {
      const language = detectLanguage(filePath, content, overrides);
      if (!pool) return Promise.resolve(extractFromSource(filePath, content, language, frameworkNames));
      return pool.requestParse({ filePath, content, language, frameworkNames });
    };

    // --- Bounded rolling-window dispatch, ordered commit ---
    // Reads stay batched/parallel; parses run concurrently across the pool; the
    // SQLite store stays on the main thread (it isn't thread-safe). Crucially we
    // COMMIT results in original file order, not parse-completion order: the
    // resolution phase (run after indexing) resolves an ambiguous reference to one
    // of several same-named candidates by the nodes' DB insertion order, so a
    // stable commit order keeps the resulting graph deterministic — byte-identical
    // to the single-worker path — instead of drifting with parse timing. The
    // `completed` buffer holds at most ~windowSize out-of-order results, so memory
    // stays bounded.
    const windowSize = pool ? Math.max(4, pool.size * 2) : 1;
    const inFlight = new Set<Promise<void>>();
    const completed = new Map<number,
      | { ok: true; filePath: string; content: string; stats: fs.Stats; result: ExtractionResult }
      | { ok: false; filePath: string; content: string; stats: fs.Stats; err: unknown }>();
    let nextSeq = 0;       // file-order sequence assigned at dispatch
    let nextToStore = 0;   // cursor: next sequence to commit
    let aborted = false;

    // Yielder for the in-order commit path: a single giant generated file's
    // store is otherwise one unyielding multi-second transaction span on the
    // main thread (5–14s single stalls measured on llvm-project), starving
    // the #850 watchdog heartbeat on slow hardware.
    const commitYield = createYielder();

    const storeResult = async (filePath: string, content: string, stats: fs.Stats, result: ExtractionResult): Promise<void> => {
      processed++;

      // WAL hard-cap backstop: between files (never mid-transaction), pause
      // the store until the off-thread checkpoint catches up. Resolves to
      // null in the normal case — a single size check, no cost.
      const bp = walBackpressure?.();
      if (bp) await bp;

      // Kernel deferred-decode results carry table sizes in kernelCounts
      // (their object arrays are empty — decode happens at the store).
      const nodeCount = result.kernelCounts?.nodes ?? result.nodes.length;
      const edgeCount = result.kernelCounts?.edges ?? result.edges.length;

      // Store: on the writer thread when active (fresh DB — bundles applied
      // in the same file order this chain dispatches them), else on the main
      // thread (SQLite connections are per-thread).
      const language = detectLanguage(filePath, content, overrides);
      if (storeWriter) {
        if (result.kernelBuffers) {
          // Buffers go to the writer as-is; the worker decodes + finalizes.
          // The main thread's only per-file work stays O(1) + the content hash.
          storeWriter.send({
            kernel: true,
            filePath,
            language,
            buffers: result.kernelBuffers,
            file: this.buildFileRecord(filePath, content, language, stats, nodeCount, result.errors),
          });
        } else {
          storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result));
        }
        await storeWriter.waitBelow(STORE_WRITER_WINDOW);
      } else {
        const materialized = materializeKernelResult(result, filePath, language);
        await this.storeExtractionResult(filePath, content, language, stats, materialized, commitYield);
      }

      if (result.errors.length > 0) {
        for (const err of result.errors) {
          if (!err.filePath) err.filePath = filePath;
        }
        errors.push(...result.errors);
      }

      if (nodeCount > 0) {
        filesIndexed++;
        totalNodes += nodeCount;
        totalEdges += edgeCount;
      } else if (result.errors.some((e) => e.severity === 'error')) {
        filesErrored++;
      } else {
        // Files with no symbols but no errors (yaml, twig, properties) are
        // tracked at the file level — count them as indexed so the CLI doesn't
        // misleadingly report "No files found to index".
        const lang = detectLanguage(filePath, content, overrides);
        if (isFileLevelOnlyLanguage(lang)) {
          filesIndexed++;
        } else {
          filesSkipped++;
        }
      }

      onProgress?.({ phase: 'parsing', current: processed, total, currentFile: filePath });
    };

    const recordParseFailure = async (filePath: string, content: string, stats: fs.Stats, err: unknown): Promise<void> => {
      await storeResult(filePath, content, stats, {
        nodes: [],
        edges: [],
        unresolvedReferences: [],
        errors: [{
          message: err instanceof Error ? err.message : String(err),
          filePath,
          severity: 'error',
          code: 'parse_error',
        }],
        durationMs: 0,
      });
    };

    // Commit buffered parses to the DB in file order, advancing the cursor over
    // contiguous completed results. Runs after each parse settles (and once more
    // after the drain). storeResult is now async (it yields between chunked
    // inserts), so commits are SERIALIZED on a promise chain — concurrent parse
    // completions append to the chain instead of interleaving mid-store, which
    // preserves both the file-order commit invariant (#1015: resolution
    // disambiguates same-named candidates by insertion order) and the
    // single-writer discipline for SQLite. Errors are recorded and re-thrown
    // at the drain, matching the old synchronous propagation.
    let flushChain: Promise<void> = Promise.resolve();
    let flushError: unknown = null;
    const flushOrdered = (): Promise<void> => {
      flushChain = flushChain.then(async () => {
        if (aborted || flushError) return;
        try {
          while (completed.has(nextToStore)) {
            const item = completed.get(nextToStore)!;
            completed.delete(nextToStore);
            nextToStore++;
            if (item.ok) await storeResult(item.filePath, item.content, item.stats, item.result);
            else await recordParseFailure(item.filePath, item.content, item.stats, item.err);
          }
        } catch (err) {
          flushError = err;
        }
      });
      return flushChain;
    };

    // Dispatch one file's parse (parses run concurrently across the pool), tagged
    // with its file-order sequence so flushOrdered commits results in order. The
    // backpressure below bounds how far parsing runs ahead of the in-order commit.
    const feed = async (filePath: string, content: string, stats: fs.Stats): Promise<void> => {
      const seq = nextSeq++;
      const p = (async () => {
        try {
          const result = await parseFile(filePath, content);
          completed.set(seq, { ok: true, filePath, content, stats, result });
        } catch (parseErr) {
          completed.set(seq, { ok: false, filePath, content, stats, err: parseErr });
        }
        flushOrdered();
      })();
      const tracked = p.finally(() => { inFlight.delete(tracked); });
      inFlight.add(tracked);
      // Backpressure on the dispatched-but-not-yet-committed count (in-flight +
      // buffered), not just in-flight: a slow file sitting at the commit cursor
      // lets later parses finish and buffer, which would otherwise grow without
      // bound. Wait for parses to settle (each may advance the cursor) until the
      // window has room. When nothing is in flight but the window is still full,
      // the async commit chain is what's behind — await it so the cursor
      // advances (buffered items hold whole file contents, so this bound is
      // load-bearing for memory).
      while (nextSeq - nextToStore >= windowSize) {
        if (inFlight.size > 0) await Promise.race(inFlight);
        else await flushOrdered();
      }
    };

    const tParseLoop = Date.now();
    for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
      if (signal?.aborted) { aborted = true; break; }

      const batch = files.slice(i, i + FILE_IO_BATCH_SIZE);

      // Read files in parallel (with path validation before any I/O)
      const fileContents = await Promise.all(
        batch.map(async (fp) => {
          try {
            // Indexing read: follow in-root symlinks the directory walk already
            // descended into (the `../` guard still applies) so files reached
            // via an in-root symlink-to-outside still index (#935).
            const fullPath = validatePathWithinRoot(this.rootDir, fp, { allowSymlinkEscape: true });
            if (!fullPath) {
              logWarn('Path traversal blocked in batch reader', { filePath: fp });
              return { filePath: fp, content: null as string | null, stats: null as fs.Stats | null, error: new Error('Path traversal blocked') };
            }
            const content = await fsp.readFile(fullPath, 'utf-8');
            const stats = await fsp.stat(fullPath);
            return { filePath: fp, content, stats, error: null as Error | null };
          } catch (err) {
            return { filePath: fp, content: null as string | null, stats: null as fs.Stats | null, error: err as Error };
          }
        })
      );

      // Dispatch each readable file into the bounded parse window; the window
      // stores results on the main thread as they arrive.
      for (const { filePath, content, stats, error } of fileContents) {
        if (signal?.aborted) { aborted = true; break; }

        if (error || content === null || stats === null) {
          processed++;
          filesErrored++;
          errors.push({
            message: `Failed to read file: ${error instanceof Error ? error.message : String(error)}`,
            filePath,
            severity: 'error',
            code: 'read_error',
          });
          onProgress?.({ phase: 'parsing', current: processed, total });
          continue;
        }

        // Honour MAX_FILE_SIZE. Without this check, vendored generated
        // headers, minified bundles, and other multi-MB files get indexed,
        // wasting WASM heap and the worker recycle budget on inputs with no
        // useful symbols. The single-file extractFile path already enforces
        // this; the bulk path used to silently skip the check.
        if (stats.size > MAX_FILE_SIZE) {
          await storeResult(filePath, content, stats, {
            nodes: [],
            edges: [],
            unresolvedReferences: [],
            errors: [{
              message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
              filePath,
              severity: 'warning',
              code: 'size_exceeded',
            }],
            durationMs: 0,
          });
          continue;
        }

        // Parse on the pool (main thread stays unblocked). Errors/timeouts are
        // handled inside feed() → recordParseFailure, feeding the retry pass.
        await feed(filePath, content, stats);
      }

      if (aborted) break;
    }

    // Drain parses still in flight (skip on abort — we tear down below instead),
    // then commit any results the cursor hasn't reached yet.
    if (!aborted) {
      await Promise.all(inFlight);
      await flushOrdered();
      if (flushError) {
        if (storeWriter) await storeWriter.close();
        throw flushError;
      }
      // All bundles are posted; wait for the writer to apply them, then close
      // its connection BEFORE any main-thread DB work below (retry pass,
      // resolution) so exactly one connection writes at a time.
      if (storeWriter) {
        try {
          await storeWriter.drain();
        } finally {
          await storeWriter.close();
          storeWriter = null;
        }
      }
    }
    if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] parse-loop: ${Date.now() - tParseLoop}ms`);

    if (signal?.aborted || aborted) {
      if (storeWriter) await storeWriter.close();
      if (pool) await pool.destroy();
      return {
        success: false,
        filesIndexed,
        filesSkipped,
        filesErrored,
        filesDiscovered: total,
        ...skipSummary(),
        nodesCreated: totalNodes,
        edgesCreated: totalEdges,
        errors: [{ message: 'Aborted', severity: 'error' }, ...errors],
        durationMs: Date.now() - startTime,
      };
    }

    // Report 100% so the progress bar doesn't hang at 99%
    onProgress?.({
      phase: 'parsing',
      current: total,
      total,
    });

    // Yield so the shimmer worker's buffered stdout writes can flush.
    // Worker thread stdout is proxied through the main thread's event loop,
    // so synchronous work here blocks the animation from rendering.
    await new Promise(resolve => setImmediate(resolve));

    // Retry pass: files that failed due to WASM memory corruption may succeed
    // on a fresh worker with a clean heap. Recycle before each attempt so
    // every file gets the absolute cleanest WASM state possible. Timeouts are
    // retried too (#1231): most are main-thread-stall artifacts, not slow
    // parses, and this pass parses one file at a time with the store strictly
    // after each parse resolves, so the stall window can't recur here.
    const retryableErrors = errors.filter(
      (e) => e.code === 'parse_error' && e.filePath &&
        (e.message.includes('Worker exited') ||
         e.message.includes('memory access out of bounds') ||
         e.message.includes('timed out'))
    );

    if (retryableErrors.length > 0 && pool) {
      log(`Retrying ${retryableErrors.length} files that failed due to WASM memory errors or timeouts...`);

      // Fresh WASM heaps for the retry phase. A retry that still crashes its
      // worker makes the pool respawn it, so later retries keep landing on clean
      // workers too.
      pool.recycleAll();

      const stillFailing: typeof retryableErrors = [];

      for (const errEntry of retryableErrors) {
        const filePath = errEntry.filePath!;
        if (signal?.aborted) break;

        let content: string;
        try {
          const fullPath = validatePathWithinRoot(this.rootDir, filePath);
          if (!fullPath) continue;
          content = await fsp.readFile(fullPath, 'utf-8');
        } catch {
          continue;
        }

        let result: ExtractionResult;
        try {
          result = await parseFile(filePath, content);
        } catch {
          stillFailing.push(errEntry);
          continue;
        }

        // The pool hands kernel results back as an undecoded buffer transport
        // (`nodes`/`edges` EMPTY, tables in kernelBuffers). The main loop
        // decodes or forwards to the store worker; this path stores directly,
        // so decode here — otherwise a kernel-language retry passes the gate
        // below via `errors.length === 0`, stores nothing, and the file is
        // permanently recorded as "(0 symbols)" with the error erased (#1541).
        const language = detectLanguage(filePath, content, overrides);
        result = materializeKernelResult(result, filePath, language);

        if (result.nodes.length > 0 || result.errors.length === 0) {
          const stats = await fsp.stat(path.join(this.rootDir, filePath));
          await this.storeExtractionResult(filePath, content, language, stats, result, commitYield);

          const idx = errors.indexOf(errEntry);
          if (idx >= 0) errors.splice(idx, 1);
          filesErrored--;
          filesIndexed++;
          totalNodes += result.nodes.length;
          totalEdges += result.edges.length;
          log(`Retry OK: ${filePath} (${result.nodes.length} nodes)`);
        }
      }

      // Last resort: for files that still crash on a clean worker, strip
      // comment-only lines to reduce WASM memory pressure. Many compiler
      // test files are 90%+ comments (CHECK directives) that don't contribute
      // code nodes but consume parser memory.
      if (stillFailing.length > 0) {
        log(`${stillFailing.length} files still failing — retrying with comments stripped...`);
        pool.recycleAll();

        for (const errEntry of stillFailing) {
          const filePath = errEntry.filePath!;
          if (signal?.aborted) break;

          let fullContent: string;
          try {
            const fullPath = validatePathWithinRoot(this.rootDir, filePath);
            if (!fullPath) continue;
            fullContent = await fsp.readFile(fullPath, 'utf-8');
          } catch {
            continue;
          }

          // Strip lines that are entirely comments (preserving line numbers
          // by replacing with empty lines so node positions stay correct)
          const stripped = fullContent
            .split('\n')
            .map(line => /^\s*\/\//.test(line) ? '' : line)
            .join('\n');

          let result: ExtractionResult;
          try {
            result = await parseFile(filePath, stripped);
          } catch {
            continue;
          }

          // Same undecoded-transport hazard as the first retry pass (#1541).
          const language = detectLanguage(filePath, fullContent, overrides);
          result = materializeKernelResult(result, filePath, language);

          if (result.nodes.length > 0 || result.errors.length === 0) {
            const stats = await fsp.stat(path.join(this.rootDir, filePath));
            await this.storeExtractionResult(filePath, fullContent, language, stats, result, commitYield);

            // Salvaged from comment-stripped source: keep a visible trace in
            // the summary instead of erasing the failure outright — the
            // stored result may be missing whatever the failing parse choked
            // on, and a silently "clean" file here is how an index quietly
            // disagrees with a later per-file sync of the same bytes (#1565).
            errEntry.severity = 'warning';
            errEntry.code = 'salvaged_stripped';
            errEntry.message = `Indexed from comment-stripped source after repeated parse failures (symbols may be incomplete until the file is re-indexed): ${errEntry.message}`;
            filesErrored--;
            filesIndexed++;
            totalNodes += result.nodes.length;
            totalEdges += result.edges.length;
            log(`Retry (stripped) OK: ${filePath} (${result.nodes.length} nodes)`);
          }
        }
      }
    }

    // Shut down the parse worker pool.
    if (pool) await pool.destroy();

    return {
      success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
      filesIndexed,
      filesSkipped,
      filesErrored,
      filesDiscovered: total,
      ...skipSummary(),
      nodesCreated: totalNodes,
      edgesCreated: totalEdges,
      errors,
      durationMs: Date.now() - startTime,
    };
  }

  /**
   * Index specific files
   */
  async indexFiles(filePaths: string[]): Promise<IndexResult> {
    const startTime = Date.now();
    const errors: ExtractionError[] = [];
    let filesIndexed = 0;
    let filesSkipped = 0;
    let filesErrored = 0;
    let totalNodes = 0;
    let totalEdges = 0;

    for (const filePath of filePaths) {
      const result = await this.indexFile(filePath);

      if (result.errors.length > 0) {
        errors.push(...result.errors);
      }

      if (result.nodes.length > 0) {
        filesIndexed++;
        totalNodes += result.nodes.length;
        totalEdges += result.edges.length;
      } else if (result.errors.some((e) => e.severity === 'error')) {
        filesErrored++;
      } else {
        const tracked = this.queries.getFileByPath(filePath);
        if (tracked && isFileLevelOnlyLanguage(tracked.language)) {
          filesIndexed++;
        } else {
          filesSkipped++;
        }
      }
    }

    return {
      success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
      filesIndexed,
      filesSkipped,
      filesErrored,
      nodesCreated: totalNodes,
      edgesCreated: totalEdges,
      errors,
      durationMs: Date.now() - startTime,
    };
  }

  /**
   * Index a single file
   */
  async indexFile(relativePath: string): Promise<ExtractionResult> {
    // Indexing read: follow in-root symlinks (the `../` guard still applies), #935.
    const fullPath = validatePathWithinRoot(this.rootDir, relativePath, { allowSymlinkEscape: true });

    if (!fullPath) {
      return {
        nodes: [],
        edges: [],
        unresolvedReferences: [],
        errors: [{ message: `Path traversal blocked: ${relativePath}`, filePath: relativePath, severity: 'error', code: 'path_traversal' }],
        durationMs: 0,
      };
    }

    // Read file content and stats
    let content: string;
    let stats: fs.Stats;
    try {
      stats = await fsp.stat(fullPath);
      content = await fsp.readFile(fullPath, 'utf-8');
    } catch (error) {
      return {
        nodes: [],
        edges: [],
        unresolvedReferences: [],
        errors: [
          {
            message: `Failed to read file: ${error instanceof Error ? error.message : String(error)}`,
            filePath: relativePath,
            severity: 'error',
            code: 'read_error',
          },
        ],
        durationMs: 0,
      };
    }

    return this.indexFileWithContent(relativePath, content, stats);
  }

  /**
   * Index a single file with pre-read content and stats.
   * Used by the parallel batch reader to avoid redundant file I/O.
   */
  async indexFileWithContent(
    relativePath: string,
    content: string,
    stats: fs.Stats
  ): Promise<ExtractionResult> {
    // Prevent `../` traversal; follow in-root symlinks like the directory walk (#935).
    const fullPath = validatePathWithinRoot(this.rootDir, relativePath, { allowSymlinkEscape: true });
    if (!fullPath) {
      logWarn('Path traversal blocked in indexFileWithContent', { relativePath });
      return {
        nodes: [],
        edges: [],
        unresolvedReferences: [],
        errors: [{ message: 'Path traversal blocked', filePath: relativePath, severity: 'error', code: 'path_traversal' }],
        durationMs: 0,
      };
    }

    const language = detectLanguage(relativePath, content, loadExtensionOverrides(this.rootDir));

    // Check file size
    if (stats.size > MAX_FILE_SIZE) {
      const result: ExtractionResult = {
        nodes: [],
        edges: [],
        unresolvedReferences: [],
        errors: [
          {
            message: `File exceeds max size (${stats.size} > ${MAX_FILE_SIZE})`,
            filePath: relativePath,
            severity: 'warning',
            code: 'size_exceeded',
          },
        ],
        durationMs: 0,
      };
      await this.storeExtractionResult(relativePath, content, language, stats, result, createYielder());
      return result;
    }

    // Detect language (honoring the project's codegraph.json extension overrides)
    if (!isLanguageSupported(language)) {
      return {
        nodes: [],
        edges: [],
        unresolvedReferences: [],
        errors: [],
        durationMs: 0,
      };
    }

    // Extract from source. Use cached framework names if indexAll has run,
    // otherwise detect on the spot so single-file re-index paths still emit
    // route nodes / middleware / etc.
    const frameworkNames = this.ensureDetectedFrameworks();
    const result = extractFromSource(relativePath, content, language, frameworkNames);

    // Store in database
    await this.storeExtractionResult(relativePath, content, language, stats, result, createYielder());

    return result;
  }

  /**
   * Store extraction result in database
   */
  /**
   * Delete file rows recorded with ZERO nodes so their files re-index.
   *
   * No extraction path stores an empty, error-free result for a
   * symbol-bearing language — even an empty file keeps its file node — so a
   * zero-node row is a wiped one (#1541: an interrupted parse's retry stored
   * an undecoded kernel transport). The wiped row's content hash matches the
   * on-disk bytes, so every hash-based reconcile skips the file forever;
   * deleting the row lets the normal add path repair it. File-level-only
   * languages (yaml, twig, properties) are left alone. Deleting a zero-node
   * row cascades nothing: it has no nodes, so no edges or refs either.
   */
  private healZeroNodeRows(): void {
    for (const f of this.queries.getAllFiles()) {
      // A zero-node row WITH recorded errors is a deliberate skip marker
      // (#1557: oversized / repeatedly-unparseable files are persisted with
      // their reason so syncs stop retrying them) — leave those alone. The
      // #1541 wipe rows are the error-FREE zero-node rows.
      if (
        f.nodeCount === 0 &&
        !isFileLevelOnlyLanguage(f.language) &&
        (f.errors === undefined || f.errors.length === 0)
      ) {
        this.queries.deleteFile(f.path);
      }
    }
  }

  private async storeExtractionResult(
    filePath: string,
    content: string,
    language: Language,
    stats: fs.Stats,
    result: ExtractionResult,
    onYield?: MaybeYield
  ): Promise<void> {
    // A kernel result can arrive as an undecoded buffer transport (empty
    // node/edge arrays, tables riding in kernelBuffers). Decode it before
    // storing — persisting the transport as-is records the file as having no
    // symbols at all (#1541). No-op for already-decoded results.
    result = materializeKernelResult(result, filePath, language);

    // Bulk inserts run in bounded sub-transactions with a yield between, so a
    // giant generated file (tens of thousands of symbols) can't block the
    // event loop — and the #850 watchdog heartbeat — for the whole store.
    // The file was NEVER one atomic transaction (each insert call has its
    // own), and the files-table record still lands last, so crash recovery
    // is unchanged: a partially-stored file has no record and re-indexes.
    const STORE_CHUNK = 2000;
    const contentHash = hashContent(content);

    // Check if file already exists and hasn't changed. A skip/failure MARKER
    // row (zero nodes + recorded errors, #1557) never blocks a store carrying
    // real content: markers are written BEFORE the retry pass under the same
    // content hash, so treating them as "no changes" would silently discard a
    // successful retry's symbols — a permanent empty file presented as
    // recovered (the #1541 wipe, reintroduced through the marker path).
    const existingFile = this.queries.getFileByPath(filePath);
    if (existingFile && existingFile.contentHash === contentHash) {
      const existingIsMarker =
        existingFile.nodeCount === 0 && (existingFile.errors?.length ?? 0) > 0;
      const incomingHasContent = result.nodes.length > 0;
      if (!existingIsMarker || !incomingHasContent) {
        return; // No changes
      }
    }

    // Re-decided on every re-index of a changed file, so a banner added (or
    // removed) by an edit is reflected on the next sync (#1500). Computed after
    // the unchanged-file early return so untouched files pay nothing.
    const generated = detectGeneratedFile(filePath, content);

    // Snapshot incoming cross-file edges BEFORE deleting this file's nodes.
    // `deleteFile` cascades to delete every edge whose source OR target is a
    // node in this file (edges.FK ... ON DELETE CASCADE). Edges whose SOURCE is
    // in this file are re-emitted by the extractor below, but edges whose SOURCE
    // is in a *different* (unchanged) file are not — they would be silently
    // dropped, which is issue #899: re-indexing a callee file severs `calls`/
    // `references` edges from callers that import it via module-attribute
    // access (`pkg.mod.fn(...)`).
    //
    // We snapshot the edge plus the target node's (name, kind) so we can
    // re-resolve to the re-indexed target's NEW id. Node ids are
    // `sha256(filePath:kind:name:line)`, so any line shift in the callee file
    // (e.g. a docstring-only edit above the symbol) changes every target id and
    // a naive re-insert by old id would silently drop every edge. Matching by
    // (filePath, kind, name) is stable across line shifts; if the symbol was
    // renamed/removed, no match is found and the edge stays dropped (correct).
    const crossFileIncomingEdges = existingFile
      ? this.queries.getCrossFileIncomingEdgesWithTarget(filePath)
      : [];

    // Delete existing data for this file
    if (existingFile) {
      this.queries.deleteFile(filePath);
    }

    // Filter out nodes with missing required fields before insertion.
    // This prevents FK violations when edges reference nodes that would
    // be silently skipped by insertNode() (see issue #42).
    const validNodes = result.nodes.filter((n) => n.id && n.kind && n.name && n.filePath && n.language);
    const insertedIds = new Set(validNodes.map((n) => n.id));
    const validEdges = result.edges.filter(
      (e) => insertedIds.has(e.source) && insertedIds.has(e.target)
    );
    const validRefs = result.unresolvedReferences
      .filter((ref) => insertedIds.has(ref.fromNodeId))
      .map((ref) => ({
        ...ref,
        filePath: ref.filePath ?? filePath,
        language: ref.language ?? language,
      }));

    // Fast path for the common case (everything fits one chunk): the whole
    // file — nodes, edges, refs, file record — lands in ONE transaction with
    // no event-loop yields in between. Giant generated files keep the chunked
    // + yielding path below so the #850 watchdog heartbeat stays serviced.
    const fitsOneChunk =
      validNodes.length <= STORE_CHUNK &&
      validEdges.length <= STORE_CHUNK &&
      validRefs.length <= STORE_CHUNK;
    if (fitsOneChunk) {
      // Snapshot/re-resolution of cross-file incoming edges (below) still runs
      // for the sync path; on a fresh bulk index crossFileIncomingEdges is [].
      this.queries.storeFileBundle({
        nodes: validNodes,
        edges: validEdges,
        refs: validRefs,
        file: {
          path: filePath,
          contentHash,
          language,
          size: stats.size,
          modifiedAt: stats.mtimeMs,
          indexedAt: Date.now(),
          nodeCount: result.nodes.length,
          errors: result.errors.length > 0 ? result.errors : undefined,
          generated,
        },
      });
      if (crossFileIncomingEdges.length > 0) {
        this.reattachCrossFileEdges(crossFileIncomingEdges, validNodes);
      }
      return;
    }

    // Insert nodes (chunked — see STORE_CHUNK above)
    for (let i = 0; i < validNodes.length; i += STORE_CHUNK) {
      this.queries.insertNodes(validNodes.slice(i, i + STORE_CHUNK));
      await onYield?.();
    }

    // Filter edges to only reference nodes that were actually inserted
    if (validEdges.length > 0) {
      for (let i = 0; i < validEdges.length; i += STORE_CHUNK) {
        this.queries.insertEdges(validEdges.slice(i, i + STORE_CHUNK));
        await onYield?.();
      }
    }

    // Re-insert cross-file incoming edges snapshotted before the delete,
    // re-resolving each edge's target to the re-indexed node's new id by
    // (filePath, kind, name). Node ids include the source line, so any line
    // shift in the callee file (e.g. a docstring-only edit above the symbol)
    // changes every target id and a naive re-insert by old id would drop them
    // all. `insertEdges` still filters to endpoints that exist. This closes
    // the #899 edge-drop on `sync`.
    //
    // Edges whose callee (target) was renamed/removed during the re-index (no
    // match in `newNodesByKindName`) are not silently dropped anymore: each is
    // resurrected as its ORIGINAL unresolved ref (stamped on the edge as
    // metadata.refName/refKind at creation) so the same sync's resolution
    // sweep can rebind it to an alternative definition elsewhere, or park it
    // as status='failed' to be retried when the symbol reappears — the
    // removal-side counterpart of #1240. Edges without refName (built before
    // the stamp existed, or synthesized) still drop silently: reconstructing
    // a ref from the target's plain name would strip receiver/qualifier
    // context and risk a rebind a full re-index would never make.
    if (crossFileIncomingEdges.length > 0) {
      this.reattachCrossFileEdges(crossFileIncomingEdges, validNodes);
    }

    // Insert unresolved references in batch with denormalized filePath/language
    for (let i = 0; i < validRefs.length; i += STORE_CHUNK) {
      this.queries.insertUnresolvedRefsBatch(validRefs.slice(i, i + STORE_CHUNK));
      await onYield?.();
    }

    // Insert file record
    const fileRecord: FileRecord = {
      path: filePath,
      contentHash,
      language,
      size: stats.size,
      modifiedAt: stats.mtimeMs,
      indexedAt: Date.now(),
      nodeCount: result.nodes.length,
      errors: result.errors.length > 0 ? result.errors : undefined,
      generated,
    };
    this.queries.upsertFile(fileRecord);
  }

  /**
   * Build one file's store bundle for the FRESH-DB path: no existing-file
   * check, no cross-file edge snapshot (both are re-index concerns — a fresh
   * database has neither). Filters mirror storeExtractionResult exactly.
   */
  /** The FileRecord for a fresh-index store (nodeCount is the PRE-filter count). */
  private buildFileRecord(
    filePath: string,
    content: string,
    language: Language,
    stats: fs.Stats,
    nodeCount: number,
    resultErrors: ExtractionResult['errors']
  ): FileRecord {
    return {
      path: filePath,
      contentHash: hashContent(content),
      language,
      size: stats.size,
      modifiedAt: stats.mtimeMs,
      indexedAt: Date.now(),
      nodeCount,
      errors: resultErrors.length > 0 ? resultErrors : undefined,
      // Decided here, once, while the content is already in memory — never at
      // query time (#1500). The header scan short-circuits on a single
      // substring test for ~every hand-written file.
      generated: detectGeneratedFile(filePath, content),
    };
  }

  private buildFreshStoreBundle(
    filePath: string,
    content: string,
    language: Language,
    stats: fs.Stats,
    result: ExtractionResult
  ): StoreBundle {
    return finalizeStoreBundle(
      result,
      filePath,
      language,
      this.buildFileRecord(filePath, content, language, stats, result.nodes.length, result.errors)
    );
  }

  /**
   * Re-attach cross-file incoming edges snapshotted before a re-index delete
   * (#899): re-resolve each edge's target to the re-indexed node's new id by
   * (kind, name); targets that vanished are resurrected as their original
   * unresolved ref (#1240's removal-side counterpart) when the edge carries
   * its refName stamp.
   */
  private reattachCrossFileEdges(
    crossFileIncomingEdges: Array<Edge & { targetKind: string; targetName: string; sourceFilePath: string; sourceLanguage: Language }>,
    validNodes: Node[]
  ): void {
    const newNodesByKindName = new Map<string, string>();
    for (const n of validNodes) {
      newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
    }
    const reinserted: Edge[] = [];
    const resurrected: UnresolvedReference[] = [];
    for (const e of crossFileIncomingEdges) {
      const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
      if (newTargetId) {
        reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
      } else {
        const ref = resurrectRefFromDroppedEdge(e);
        if (ref) resurrected.push(ref);
      }
    }
    if (reinserted.length > 0) {
      this.queries.insertEdges(reinserted);
    }
    if (resurrected.length > 0) {
      this.queries.insertUnresolvedRefsBatch(resurrected);
    }
  }

  /**
   * Re-open, for re-resolution, every resolution edge whose answer this sync
   * may have changed — the fix for index drift (CG-33).
   *
   * Incremental sync re-resolves only the references IN the changed files, but
   * resolution's answer is a function of the WHOLE graph: a reference binds to
   * one of the same-named definitions project-wide, so adding or removing a
   * definition of `pct` can change which `pct` every other file's `pct(...)`
   * should bind to. Those other files are never revisited, and their references
   * resolved successfully once and were deleted from `unresolved_refs`, so
   * nothing existed to revisit them with — the index kept an answer that was
   * correct against an older graph. Measured on codegraph's own long-lived
   * index: 4.3% of distinct edges differed from a clean rebuild, in BOTH
   * directions, overwhelmingly `calls`. See docs/benchmarks/index-drift-cg33.md.
   *
   * This deletes each affected edge and re-inserts it as the reference that
   * created it (the refName/refKind stamp), status='pending', for the sync's
   * resolution sweep to bind against the post-sync graph — the same input a
   * full rebuild resolves from, which is what makes the two converge.
   *
   * Deliberately conservative in three ways, because a wrong deletion is a
   * permanent edge loss while a missed rebind is only residual drift:
   * - an edge with no refName stamp (synthesized, or built by an engine older
   *   than the stamp) is left ALONE rather than reconstructed from the target's
   *   plain name, same rule as `resurrectRefFromDroppedEdge`;
   * - edges whose source is in a file this sync already re-extracted are
   *   skipped — their references were re-resolved from scratch moments ago;
   * - very common names are skipped by the per-name ceiling in
   *   `getResolutionEdgesByTargetName`.
   *
   * Returns the number of references resurrected.
   */
  resurrectStaleResolutionEdges(definitionDelta: string[], changedFilePaths: string[]): number {
    if (definitionDelta.length === 0) return 0;
    const alreadyFresh = new Set(changedFilePaths);
    const candidates = this.queries.getResolutionEdgesByTargetName(definitionDelta);

    const edgeIds: number[] = [];
    const refs: UnresolvedReference[] = [];
    for (const e of candidates) {
      if (alreadyFresh.has(e.sourceFilePath)) continue;
      const ref = resurrectRefFromDroppedEdge(e);
      if (!ref) continue; // no stamp — never delete what we cannot restore
      edgeIds.push(e.edgeId);
      refs.push(ref);
    }
    if (refs.length === 0) return 0;

    // Delete first. The sweep re-inserts whichever edge resolution now picks,
    // and `insertEdges` is INSERT OR IGNORE against idx_edges_identity — so a
    // rebind to the same target is a clean no-op, but leaving the old row in
    // place for a rebind ELSEWHERE would keep both, turning drift into
    // duplication.
    this.queries.replaceResolutionEdgesWithUnresolvedRefs(edgeIds, refs);
    return refs.length;
  }

  /**
   * Sync the index with the current file state.
   *
   * Change detection is filesystem-based, never git: a (size, mtime) stat
   * pre-filter skips unchanged files, then a content-hash compare confirms real
   * changes. This works in non-git projects and catches committed changes from
   * `git pull`/`checkout`/`merge`/`rebase` that `git status` cannot see.
   */
  async sync(
    onProgress?: (progress: IndexProgress) => void,
    /**
     * Watcher fast path: the exact project-relative paths the OS reported as
     * changed. When provided, reconciliation runs over ONLY these paths —
     * per-path logic identical to the full walk (stat pre-filter, hash
     * confirm, the #1240 removal/resurrection flow) — skipping the O(repo)
     * scan and tracked-load. Callers must pass undefined whenever the change
     * set is not exactly known (directory removals, event overflow): the full
     * scan-diff remains the ground truth those cases need (#1285).
     */
    scopedPaths?: string[],
    /**
     * Writer-side WAL pressure valve (#1539). Called after every changed file
     * is stored, when no extraction transaction is open, so a checkpoint can
     * safely catch up before the next file grows the WAL further.
     */
    backpressure?: () => Promise<void> | null
  ): Promise<SyncResult> {
    await initGrammars(); // Initialize WASM runtime (grammars loaded lazily below)
    const startTime = Date.now();
    let filesChecked = 0;
    let filesAdded = 0;
    let filesModified = 0;
    let filesRemoved = 0;
    let nodesUpdated = 0;
    const changedFilePaths: string[] = [];
    // `file\0name` definition pairs for the files this sync touches, sampled
    // BEFORE their nodes are replaced/deleted. Compared against the post-store
    // pairs below to derive `definitionDelta` (CG-33).
    const pairsBefore = new Set<string>();

    onProgress?.({
      phase: 'scanning',
      current: 0,
      total: 0,
    });

    const filesToIndex: string[] = [];
    const failedFilePaths: string[] = [];
    // === Filesystem reconcile (git-independent) ===
    // The source of truth for "what changed" is the filesystem vs the indexed
    // state — never git. We enumerate the current source files and reconcile
    // each against the DB. A cheap (size, mtime) stat pre-filter skips unchanged
    // files without reading or hashing them, so the expensive read+hash+parse
    // only runs for files that actually changed. This catches edits/adds/deletes
    // whether or not the project uses git, and crucially also catches committed
    // changes from `git pull`/`checkout`/`merge`/`rebase` — which `git status`
    // cannot see, because the working tree is clean afterward.
    const tSyncScan = Date.now();
    let currentFiles: string[];
    let trackedFiles: FileRecord[];
    if (scopedPaths && scopedPaths.length > 0) {
      // Scoped reconcile: stat only the reported paths. filesChecked counts
      // the PATHS examined (not the files found) — it must stay non-zero even
      // when every scoped path was a deletion, because CodeGraph.watch()
      // reads `filesChecked === 0 && durationMs === 0` as the
      // lock-unavailable signature (#449).
      const unique = [...new Set(scopedPaths)];
      // A scoped path is "present" only if it exists AND is in scope — the
      // same two gates the full walk applies (source extension, scope
      // matcher). Without the scope gate a caller's stale view of scope
      // leaked straight into the index: the watcher re-parsed a file the
      // user had just excluded in `codegraph.json` while `codegraph sync`
      // removed it (#1590). Out-of-scope paths fall out of `currentFiles`,
      // so a tracked one takes the removal branch below, exactly as a full
      // sync would treat it. (`include`-forced paths pass: ScopeIgnore
      // applies the include precedence itself.)
      const scope = this.scopedSyncMatcher();
      const overrides = loadExtensionOverrides(this.rootDir);
      currentFiles = unique.filter(
        (p) =>
          isSourceFile(p, overrides) &&
          !scope.ignores(p) &&
          fs.existsSync(path.join(this.rootDir, p))
      );
      trackedFiles = [];
      for (const p of unique) {
        const rec = this.queries.getFileByPath(p);
        if (rec) trackedFiles.push(rec);
      }
      filesChecked = unique.length;
      if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-scoped: ${Date.now() - tSyncScan}ms (${unique.length} paths, ${trackedFiles.length} tracked)`);
    } else {
      // Full reconcile: drop the memoized scope matcher so a nested
      // `.gitignore` / exclude-standard change that forced this full sync is
      // visible to the next scoped sync (#1728).
      this.scopedMatcher = null;
      currentFiles = await scanDirectoryAsync(this.rootDir);
      if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-scan: ${Date.now() - tSyncScan}ms (${currentFiles.length} files)`);
      filesChecked = currentFiles.length;

      // Full reconcile only (scoped syncs must not touch rows outside their
      // scope): drop zero-node rows so the wiped files re-index as adds below.
      this.healZeroNodeRows();

      const tTracked = Date.now();
      trackedFiles = this.queries.getAllFiles();
      if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-tracked-load: ${Date.now() - tTracked}ms (${trackedFiles.length} tracked)`);
    }
    const currentSet = new Set(currentFiles);
    const trackedMap = new Map<string, FileRecord>();
    for (const f of trackedFiles) {
      trackedMap.set(f.path, f);
    }

    // Removals: tracked in the DB but no longer a present source file. Check the
    // filesystem directly — `scanDirectory` (via `git ls-files`) still lists a
    // file deleted from disk but not yet staged, so set membership alone misses it.
    // `reconcileChecks` drives the cooperative yield shared with the adds/mods loop
    // below (see SYNC_RECONCILE_YIELD_INTERVAL / issue #905).
    let reconcileChecks = 0;
    for (const tracked of trackedFiles) {
      if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) {
        // Before the cascade deletes them, resurrect incoming cross-file
        // resolution edges as their original refs (#1240 removal case): the
        // callers live in files this sync will NOT revisit, so this is their
        // only chance to rebind to an alternative definition — or to park as
        // failed until the symbol reappears somewhere. (A deleted file whose
        // CALLERS are also being deleted is fine: their nodes cascade later
        // in this loop and take the resurrected rows with them.)
        // Every name this file defined is about to stop existing here, which
        // narrows the candidate set for that name repo-wide (CG-33).
        for (const pair of this.queries.getNodeNamePairsByFiles([tracked.path])) pairsBefore.add(pair);
        const incoming = this.queries.getCrossFileIncomingEdgesWithTarget(tracked.path);
        if (incoming.length > 0) {
          const resurrected = incoming
            .map((e) => resurrectRefFromDroppedEdge(e))
            .filter((r): r is UnresolvedReference => r !== null);
          if (resurrected.length > 0) {
            this.queries.insertUnresolvedRefsBatch(resurrected);
          }
        }
        this.queries.deleteFile(tracked.path);
        filesRemoved++;
      }
      if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
        await new Promise<void>((resolve) => setImmediate(resolve));
      }
    }

    // Adds / modifications.
    for (const filePath of currentFiles) {
      // Same cooperative yield as the removals loop — this is the other O(files)
      // synchronous-stat loop that wedges the main thread on a large repo (#905).
      // Yield at the top of the body so the `continue` fast-paths below still hit it.
      if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) {
        await new Promise<void>((resolve) => setImmediate(resolve));
      }
      const fullPath = path.join(this.rootDir, filePath);
      const tracked = trackedMap.get(filePath);

      // Cheap pre-filter: an already-indexed file whose size AND mtime both match
      // the DB is unchanged — skip it without reading or hashing. (A content
      // change that preserves both exactly is the blind spot every mtime-based
      // incremental tool accepts; `index --force` is the escape hatch. Git bumps
      // mtime on every file it writes during checkout/merge, so pulls are caught.)
      if (tracked) {
        try {
          const stat = fs.statSync(fullPath);
          if (stat.size === tracked.size && Math.floor(stat.mtimeMs) === Math.floor(tracked.modifiedAt)) {
            continue;
          }
        } catch (error) {
          logDebug('Skipping unstattable file during sync', { filePath, error: String(error) });
          failedFilePaths.push(filePath);
          continue;
        }
      }

      // New, or size/mtime changed — read + hash to confirm a real content change.
      let content: string;
      try {
        content = fs.readFileSync(fullPath, 'utf-8');
      } catch (error) {
        logDebug('Skipping unreadable file during sync', { filePath, error: String(error) });
        failedFilePaths.push(filePath);
        continue;
      }
      const contentHash = hashContent(content);

      if (!tracked) {
        filesToIndex.push(filePath);
        changedFilePaths.push(filePath);
        filesAdded++;
      } else if (tracked.contentHash !== contentHash) {
        filesToIndex.push(filePath);
        changedFilePaths.push(filePath);
        filesModified++;
      }
    }

    // Sampled here — after the add/modify classification, before any file is
    // re-extracted — because `storeExtractionResult` deletes a file's nodes
    // before inserting the new ones, so this is the last point the pre-edit
    // definition set is readable (CG-33).
    if (filesToIndex.length > 0) {
      for (const pair of this.queries.getNodeNamePairsByFiles(filesToIndex)) pairsBefore.add(pair);
    }

    // Load only grammars needed for changed files
    if (filesToIndex.length > 0) {
      const overrides = loadExtensionOverrides(this.rootDir);
      await loadGrammarsForLanguages(preloadLanguagesForFiles(filesToIndex, overrides));
    }

    // Index changed files
    const total = filesToIndex.length;
    for (let i = 0; i < filesToIndex.length; i++) {
      const filePath = filesToIndex[i]!;
      onProgress?.({
        phase: 'parsing',
        current: i + 1,
        total,
        currentFile: filePath,
      });

      const result = await this.indexFile(filePath);
      if (result.errors.some(e => e.severity === 'error')) failedFilePaths.push(filePath);
      nodesUpdated += result.nodes.length;

      const pause = backpressure?.();
      if (pause) await pause;
    }

    // Names whose definition set this sync changed: a `file\0name` pair present
    // before but not after (removed/renamed away) or after but not before
    // (added). A pair on both sides is untouched as far as resolution's
    // candidate set is concerned — only its node id moved, which
    // reattachCrossFileEdges already follows — so an edit that only changes
    // bodies yields an empty delta and no downstream rebind work (CG-33).
    //
    // Compared per FILE, not as one name set over the whole batch: a commit
    // that adds `collect` to a new file while an unrelated changed file already
    // defined `collect` must still flag the name, and a bare name set cancels
    // exactly that case out. That miss left the largest residual class in the
    // first measurement of this fix.
    const pairsAfter = this.queries.getNodeNamePairsByFiles(filesToIndex);
    const deltaNames = new Set<string>();
    const nameOf = (pair: string) => pair.slice(pair.indexOf('\0') + 1);
    for (const pair of pairsBefore) if (!pairsAfter.has(pair)) deltaNames.add(nameOf(pair));
    for (const pair of pairsAfter) if (!pairsBefore.has(pair)) deltaNames.add(nameOf(pair));
    const definitionDelta = [...deltaNames];

    return {
      filesChecked,
      filesAdded,
      filesModified,
      filesRemoved,
      nodesUpdated,
      durationMs: Date.now() - startTime,
      changedFilePaths: changedFilePaths.length > 0 ? changedFilePaths : undefined,
      ...(failedFilePaths.length > 0 ? { failedFilePaths } : {}),
      definitionDelta: definitionDelta.length > 0 ? definitionDelta : undefined,
    };
  }

  private indexedDirtyPaths(stamp: string | null): string[] | null {
    try {
      const state = JSON.parse(this.queries.getMetadata('indexed_dirty_paths') ?? 'null');
      if (!state || state.commit !== (stamp ?? '') || !Array.isArray(state.paths)) return null;
      if (!state.paths.every((p: unknown) => typeof p === 'string' && p.length > 0 &&
        !path.isAbsolute(p) && !p.split('/').includes('..'))) return null;
      return state.paths;
    } catch { return null; }
  }

  /** Capture before file reads. In-flight/failed full writes must not claim freshness. */
  beginGitIndexState(full: boolean): { head: string; stamp: string; dirty: string[] | null } {
    const head = getGitHeadSha(this.rootDir) ?? '';
    const stamp = this.queries.getMetadata(INDEXED_AT_COMMIT_KEY) ?? '';
    const prior = this.indexedDirtyPaths(stamp);
    const status = getGitChangedFiles(this.rootDir);
    const dirty = status ? [...new Set([
      ...(full ? [] : prior ?? []), ...status.added, ...status.modified, ...status.deleted,
    ])] : null;
    if (full || prior === null || dirty === null) {
      this.queries.setMetadata(INDEXED_AT_COMMIT_KEY, '');
      this.queries.setMetadata('indexed_dirty_paths', '');
    } else {
      // Scoped writes leave the commit alone and retain dirty paths before the
      // first write, so a crash cannot forget an indexed uncommitted edit.
      this.queries.setMetadata('indexed_dirty_paths', JSON.stringify({ commit: stamp, paths: dirty }));
    }
    return { head, stamp, dirty };
  }

  finishGitIndexState(snapshot: { head: string; stamp: string; dirty: string[] | null }, full: boolean, retries: string[] = []): void {
    const after = getGitChangedFiles(this.rootDir);
    if (!snapshot.dirty || !after) return;
    const commit = full ? snapshot.head : snapshot.stamp;
    const paths = [...new Set([...snapshot.dirty, ...after.added, ...after.modified, ...after.deleted, ...retries])].sort();
    // The embedded commit makes a torn pair fail closed: readers reject a dirty
    // set that doesn't match the separately stored commit. Write the set first.
    this.queries.setMetadata('indexed_dirty_paths', JSON.stringify({ commit, paths }));
    if (full) this.queries.setMetadata(INDEXED_AT_COMMIT_KEY, commit);
  }

  /**
   * Get files that have changed since last index.
   * Uses git status as a fast path when available, falling back to full scan.
   */
  getChangedFiles(): { added: string[]; modified: string[]; removed: string[] } {
    // The commit this index was last brought up to date at. Absent on an index
    // built before stamping existed — getGitChangedFiles then declines the fast
    // path and the full scan below answers correctly, once, until a sync or a
    // full index writes the stamp. (#1829)
    let sinceCommit: string | null = null;
    try { sinceCommit = this.queries.getMetadata(INDEXED_AT_COMMIT_KEY) ?? null; } catch { /* advisory */ }
    const dirtyPaths = this.indexedDirtyPaths(sinceCommit);
    const gitChanges = dirtyPaths !== null && canTrustGitFastPath(this.rootDir, sinceCommit)
      ? getGitChangedFiles(this.rootDir, sinceCommit)
      : null;

    if (gitChanges) {
      // === Git fast path ===
      const added: string[] = [];
      const modified: string[] = [];
      const removed: string[] = [];

      // Git supplies candidates, never the verdict. A committed deletion may
      // have been recreated locally; a previously indexed dirty path may have
      // vanished from git status after restore. Classify current disk vs DB once.
      const candidates = new Set([...gitChanges.deleted, ...gitChanges.modified, ...gitChanges.added, ...dirtyPaths!]);
      const scope = this.scopedSyncMatcher();
      const overrides = loadExtensionOverrides(this.rootDir);
      for (const filePath of candidates) {
        const tracked = this.queries.getFileByPath(filePath);
        const fullPath = path.join(this.rootDir, filePath);
        if (!isSourceFile(filePath, overrides) || scope.ignores(filePath) || !fs.existsSync(fullPath)) {
          if (tracked) removed.push(filePath);
          continue;
        }
        let content: string;
        try { content = fs.readFileSync(fullPath, 'utf-8'); }
        catch (error) {
          logDebug('Skipping unreadable file while detecting changes', { filePath, error: String(error) });
          continue;
        }
        if (!tracked) added.push(filePath);
        else if (tracked.contentHash !== hashContent(content)) modified.push(filePath);
      }

      return { added, modified, removed };
    }

    // === Fallback: full scan (non-git project or git failure) ===
    const currentFiles = new Set(scanDirectory(this.rootDir));
    const trackedFiles = this.queries.getAllFiles();

    // Build Map for O(1) lookups
    const trackedMap = new Map<string, FileRecord>();
    for (const f of trackedFiles) {
      trackedMap.set(f.path, f);
    }

    const added: string[] = [];
    const modified: string[] = [];
    const removed: string[] = [];

    // Find removed files
    for (const tracked of trackedFiles) {
      if (!currentFiles.has(tracked.path)) {
        removed.push(tracked.path);
      }
    }

    // Find added and modified files
    for (const filePath of currentFiles) {
      const fullPath = path.join(this.rootDir, filePath);
      let content: string;
      try {
        content = fs.readFileSync(fullPath, 'utf-8');
      } catch (error) {
        logDebug('Skipping unreadable file while detecting changes', { filePath, error: String(error) });
        continue;
      }

      const contentHash = hashContent(content);
      const tracked = trackedMap.get(filePath);

      if (!tracked) {
        added.push(filePath);
      } else if (tracked.contentHash !== contentHash) {
        modified.push(filePath);
      }
    }

    return { added, modified, removed };
  }
}

// Re-export useful types and functions
export { extractFromSource } from './tree-sitter';
export { detectLanguage, isSourceFile, isLanguageSupported, isGrammarLoaded, getSupportedLanguages, initGrammars, loadGrammarsForLanguages, loadAllGrammars } from './grammars';