feat(resolution): memory-aware, cgroup-honest worker-pool sizing + CODEGRAPH_RESOLVE_WORKERS (#1333)

Pool sizing used os.cpus().length, which enumerates the HOST's CPUs: inside
a 2-CPU cpuset it sized 6 resolver workers (the §7a.1 false-'sequential'
premise) and 8 parse workers, and at true 8-core concurrency six ~1GB
workers OOM-killed a 7GB container (oom_kill=5) mid-synthesis — sizing had
no memory term and no override knob.

resolvePoolSize (pure, matrix-tested): explicit CODEGRAPH_RESOLVE_WORKERS
override (0 disables, cap 16); CPU term max(2, min(availableParallelism-1,
6)) — cpuset-honest, floored at 2 so true 2-core boxes keep pooled
synthesis's ~2×; memory term floor(budget*0.7 / clamp(0.2*dbSize, 256MB,
1.5GB)) with budget = min(freemem, cgroup v2/v1 headroom). Parse pool's
core input switches to availableParallelism. Dev machines are unchanged
(still 6 workers); the 8c/7GB kernel-scale container now sizes 4.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

CHANGELOG.md

@@ -20,6 +20,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - Indexing is significantly faster — a fresh `codegraph init` on a medium TypeScript project takes about a third less wall-clock time, with the same graph produced byte-for-byte. The gains come from batching database writes, storing files on a dedicated writer thread, memoizing repeated import-resolution lookups, skipping per-row search-index maintenance during the bulk build (rebuilt once at the end), and — on completely fresh databases only — deferring disk durability until the index completes, since an interrupted first index is simply re-run. Set `CODEGRAPH_NO_FAST_INIT=1` to keep full crash-durability during the initial build, or `CODEGRAPH_NO_STORE_WORKER=1` to store on the main thread.
 - `codegraph install` and `codegraph upgrade` now offer CodeGraph Pro beta access after finishing — answer yes, type your email, and you join the same waitlist as the getcodegraph.com homepage form. Strictly opt-in and asked at most once per machine total: nothing is sent unless you say yes and enter an email, either answer is remembered so no later install or upgrade ever re-asks, and non-interactive runs (`--yes`, scripts, CI) never see the question.
 - Every release is now cryptographically verifiable: npm packages publish with npm provenance (the "Provenance" badge on npmjs.com, proving each version was built by this repository's release workflow from a specific commit), and the GitHub Release bundles carry signed build attestations you can check with `gh attestation verify <file> -R colbymchenry/codegraph`.
+- Indexing inside CPU- or memory-limited containers (Docker, CI runners) now sizes its worker pools from the container's actual allowance instead of the host machine's, and giant codebases no longer balloon temporary database files during indexing (previously tens of GB of transient disk on Linux-kernel-scale projects). Together these prevent out-of-memory and out-of-disk failures on constrained machines; set `CODEGRAPH_RESOLVE_WORKERS` to override the resolution worker count explicitly.
 
 ### Fixes
 

__tests__/resolver-pool-sizing.test.ts

@@ -0,0 +1,79 @@
+/**
+ * Resolver-pool sizing (§7a.1 P1.2): cgroup-honest CPU term + memory-aware
+ * cap + the CODEGRAPH_RESOLVE_WORKERS override. resolvePoolSize is pure —
+ * these pin the whole decision matrix, including the two failure modes the
+ * measurement round exposed: os.cpus() cpuset-blindness (6 workers inside a
+ * 2-CPU container) and memory-blind sizing (six ~1GB workers OOM-killing a
+ * 7GB container at true 8-core concurrency).
+ */
+import { describe, it, expect } from 'vitest';
+import { ResolverPool } from '../src/resolution/resolver-pool';
+import { cgroupMemoryAvailable, memoryBudgetBytes } from '../src/resolution/memory-budget';
+
+const GB = 1024 * 1024 * 1024;
+const MB = 1024 * 1024;
+
+function size(opts: Partial<Parameters<typeof ResolverPool.resolvePoolSize>[0]>): number | null {
+  return ResolverPool.resolvePoolSize({
+    availableParallelism: 8,
+    memoryBudget: 16 * GB,
+    dbSizeBytes: 200 * MB,
+    ...opts,
+  });
+}
+
+describe('ResolverPool.resolvePoolSize', () => {
+  it('big dev box: CPU-capped at the long-standing 6', () => {
+    expect(size({})).toBe(6);
+    expect(size({ availableParallelism: 11 })).toBe(6);
+  });
+
+  it('true 2-core box keeps a 2-worker pool (the pooled-synthesis 2×)', () => {
+    expect(size({ availableParallelism: 2, memoryBudget: 6 * GB })).toBe(2);
+  });
+
+  it('kernel-scale DB in a 7GB container: memory term shrinks the pool below the OOM line', () => {
+    // 4.6GB DB → ~940MB/worker estimate; 5.5GB headroom × 0.7 ≈ 3.85GB → 4 workers.
+    const s = size({ availableParallelism: 8, memoryBudget: 5.5 * GB, dbSizeBytes: 4.6 * GB });
+    expect(s).toBe(4);
+    expect(s!).toBeLessThan(6);
+  });
+
+  it('per-worker estimate is floored (small DBs) and capped (huge DBs)', () => {
+    // Small DB: floor 256MB/worker — memory cap = 16GB*0.7/256MB = 43 → CPU wins.
+    expect(size({ dbSizeBytes: 10 * MB })).toBe(6);
+    // Monster DB: cap 1.5GB/worker — 16GB*0.7/1.5GB = 7 → CPU still wins at 6.
+    expect(size({ dbSizeBytes: 40 * GB })).toBe(6);
+    // Same monster DB, tight memory: 4GB*0.7/1.5GB = 1 → below 2 → no pool.
+    expect(size({ dbSizeBytes: 40 * GB, memoryBudget: 4 * GB })).toBeNull();
+  });
+
+  it('starved memory disables the pool entirely', () => {
+    expect(size({ memoryBudget: 512 * MB, dbSizeBytes: 4 * GB })).toBeNull();
+  });
+
+  it('CODEGRAPH_RESOLVE_WORKERS overrides everything: 0 disables, values clamp at 16', () => {
+    expect(size({ explicit: '0' })).toBeNull();
+    expect(size({ explicit: '3', memoryBudget: 512 * MB })).toBe(3); // override skips the memory term
+    expect(size({ explicit: '64' })).toBe(16);
+    expect(size({ explicit: 'nonsense' })).toBe(6); // unparseable → computed path
+  });
+});
+
+describe('memory budget helpers', () => {
+  it('memoryBudgetBytes is positive and finite on every platform', () => {
+    const b = memoryBudgetBytes();
+    expect(b).toBeGreaterThan(0);
+    expect(Number.isFinite(b)).toBe(true);
+  });
+
+  it('cgroupMemoryAvailable is null when uncontained (non-Linux) and never throws', () => {
+    const v = cgroupMemoryAvailable();
+    if (process.platform !== 'linux') {
+      expect(v).toBeNull();
+    } else {
+      // Containerized CI: either uncontained (null) or a sane byte count.
+      expect(v === null || (v >= 0 && Number.isFinite(v))).toBe(true);
+    }
+  });
+});

src/extraction/index.ts

@@ -1615,8 +1615,12 @@ export class ExtractionOrchestrator {
     let pool: ParseWorkerPool | null = null;
     if (useWorker) {
       // CODEGRAPH_PARSE_WORKERS: explicit worker count; 1 = the old single-worker
-      // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8).
-      const poolSize = resolveParsePoolSize(process.env.CODEGRAPH_PARSE_WORKERS, os.cpus().length);
+      // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8),
+      // with cores from availableParallelism — cpuset/affinity-honest, where
+      // os.cpus() enumerates the host's CPUs and spawned 8 wasm workers (and
+      // their grammar heaps) inside a 2-CPU container for zero extra
+      // throughput (§7a.1).
+      const poolSize = resolveParsePoolSize(process.env.CODEGRAPH_PARSE_WORKERS, os.availableParallelism());
       // Read each needed grammar's WASM ONCE here and hand the bytes to every
       // worker, so spawns/respawns load grammars from memory instead of
       // re-reading them from disk (#1231: on an HDD, respawn re-reads amplify

src/resolution/memory-budget.ts

@@ -0,0 +1,59 @@
+/**
+ * Memory headroom for worker-pool sizing — cgroup-honest on Linux.
+ *
+ * `os.freemem()` reads /proc/meminfo, which inside a container reports the
+ * HOST's (or VM's) memory, not the cgroup's — the same blindness os.cpus()
+ * has for cpusets. A resolver pool sized by cores alone OOM-killed a
+ * kernel-scale index in a 7GB-capped container (migration plan §7a.1:
+ * oom_kill=5, six ~1GB workers at true 8-core concurrency), so pool sizing
+ * combines a CPU term with the memory headroom this module reports.
+ */
+
+import * as fs from 'fs';
+import * as os from 'os';
+
+/** Parse a cgroup value file: numeric bytes, or null for absent/'max'. */
+function readCgroupBytes(path: string): number | null {
+  try {
+    const raw = fs.readFileSync(path, 'utf8').trim();
+    if (raw === 'max') return null;
+    const n = Number.parseInt(raw, 10);
+    return Number.isFinite(n) && n >= 0 ? n : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Available headroom under the cgroup memory limit (v2 then v1), or null
+ * when uncontained (no limit, non-Linux, or unreadable). Never throws.
+ */
+export function cgroupMemoryAvailable(): number | null {
+  if (process.platform !== 'linux') return null;
+  // v2 unified hierarchy
+  const v2Max = readCgroupBytes('/sys/fs/cgroup/memory.max');
+  if (v2Max !== null) {
+    const current = readCgroupBytes('/sys/fs/cgroup/memory.current') ?? 0;
+    return Math.max(0, v2Max - current);
+  }
+  // v1
+  const v1Limit = readCgroupBytes('/sys/fs/cgroup/memory/memory.limit_in_bytes');
+  // v1 reports "no limit" as a huge sentinel (~PAGE_COUNTER_MAX); treat
+  // anything at or beyond half the address-space-ish range as uncontained.
+  if (v1Limit !== null && v1Limit < 2 ** 60) {
+    const usage = readCgroupBytes('/sys/fs/cgroup/memory/memory.usage_in_bytes') ?? 0;
+    return Math.max(0, v1Limit - usage);
+  }
+  return null;
+}
+
+/**
+ * The budget pool sizing divides: the smaller of system free memory and the
+ * cgroup headroom (when contained). Conservative by construction — both
+ * numbers shrink as the process itself grows.
+ */
+export function memoryBudgetBytes(): number {
+  const free = os.freemem();
+  const cgroup = cgroupMemoryAvailable();
+  return cgroup === null ? free : Math.min(free, cgroup);
+}

src/resolution/resolver-pool.ts

@@ -15,6 +15,7 @@ import * as path from 'path';
 import * as os from 'os';
 import type { Edge, UnresolvedReference } from '../types';
 import type { ResolvedRef, UnresolvedRef } from './types';
+import { memoryBudgetBytes } from './memory-budget';
 
 /** One synthesis pass's output: its edge list + worker-measured wall clock. */
 export interface SynthPassResult {
@@ -64,17 +65,70 @@ export class ResolverPool {
   private synthWaiters = new Map<number, { resolve: (r: SynthPassResult) => void; reject: (e: Error) => void }>();
   private failed: Error | null = null;
 
+  /**
+   * Pool size from CPU headroom, memory headroom, and the explicit override.
+   * Pure — every input injected — so the whole matrix is unit-testable.
+   *
+   * CPU term: `availableParallelism` (cpuset/affinity-honest — `os.cpus()`
+   * enumerates the host's CPUs and sized SIX workers inside a 2-CPU cpuset,
+   * §7a.1's false-premise finding), minus one for the persisting main thread,
+   * floored at 2 so a true 2-core box keeps the pool's ~2× on synthesis,
+   * capped at the long-standing 6.
+   *
+   * Memory term: workers hold real heap at scale (~1GB each against a 4.6GB
+   * kernel-scale DB — six of them OOM-killed a 7GB container once real
+   * 8-core concurrency let them peak simultaneously). Estimate per-worker
+   * cost from the DB size, keep 30% of the budget for the main thread, and
+   * let the smaller term win. Below 2 workers the pool isn't worth its boot
+   * cost — callers get null and stay sequential.
+   */
+  static resolvePoolSize(opts: {
+    explicit?: string;
+    availableParallelism: number;
+    memoryBudget: number;
+    dbSizeBytes: number;
+  }): number | null {
+    if (opts.explicit !== undefined && opts.explicit !== '') {
+      const n = Number.parseInt(opts.explicit, 10);
+      if (Number.isFinite(n)) {
+        if (n <= 0) return null;
+        return Math.min(n, 16);
+      }
+    }
+    const cpuCap = Math.max(2, Math.min(opts.availableParallelism - 1, 6));
+    const perWorker = Math.min(Math.max(opts.dbSizeBytes * 0.2, 256 * 1024 * 1024), 1.5 * 1024 * 1024 * 1024);
+    const memCap = Math.floor((opts.memoryBudget * 0.7) / perWorker);
+    const size = Math.min(cpuCap, memCap);
+    return size >= 2 ? size : null;
+  }
+
   /**
    * Create a pool when the compiled worker exists (absent when running from
    * source in tests → callers use the sequential path), the kill switch is
-   * off, and the machine has cores to spare. Returns null otherwise.
+   * off, and the machine has the cores AND memory to carry it. Returns null
+   * otherwise. `CODEGRAPH_RESOLVE_WORKERS` overrides the computed size
+   * (0 disables the pool; values are capped at 16).
    */
   static tryCreate(dbPath: string, projectRoot: string): ResolverPool | null {
     if (process.env.CODEGRAPH_NO_PARALLEL_RESOLVE === '1') return null;
     const workerScript = path.join(__dirname, 'resolver-worker.js');
     if (!fs.existsSync(workerScript)) return null;
-    const size = Math.max(1, Math.min(os.cpus().length - 2, 6));
-    if (size < 2) return null;
+    let dbSizeBytes = 0;
+    try {
+      dbSizeBytes = fs.statSync(dbPath).size;
+    } catch { /* fresh/missing file — the 256MB per-worker floor applies */ }
+    const size = ResolverPool.resolvePoolSize({
+      explicit: process.env.CODEGRAPH_RESOLVE_WORKERS,
+      availableParallelism: os.availableParallelism(),
+      memoryBudget: memoryBudgetBytes(),
+      dbSizeBytes,
+    });
+    if (size === null) return null;
+    if (process.env.CODEGRAPH_SYNTH_TIMINGS) {
+      console.error(
+        `[pool-timing] pool size=${size} (ap=${os.availableParallelism()} budget=${Math.round(memoryBudgetBytes() / 1024 / 1024)}MB db=${Math.round(dbSizeBytes / 1024 / 1024)}MB)`
+      );
+    }
     try {
       return new ResolverPool(workerScript, dbPath, projectRoot, size);
     } catch {