parse-run.mjs 70 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374
  1. #!/usr/bin/env node
  2. // Parse Claude Code stream-json run log(s): tool-call sequence, token usage, and
  3. // RESIDUAL CONTEXT OCCUPANCY — how many tokens of the context window each tool
  4. // family's responses still occupy when the run ends.
  5. //
  6. // Usage: parse-run.mjs <run.jsonl> [run.t2.jsonl ...] [--envelope] [--answer <glob>]...
  7. // Multiple files = one multi-turn session's segments, IN ORDER (run-all.sh
  8. // writes run-<label>.jsonl, run-<label>.t2.jsonl, … for a `Q1||Q2||Q3` set).
  9. // `--resume` does not replay prior messages, so the segments concatenate
  10. // cleanly and token accounting carries across the boundary.
  11. //
  12. // Every run also reports EXPLORE SUFFICIENCY — each codegraph_explore call
  13. // bucketed by what the agent did next (see classifySufficiency) — and EXPLORE
  14. // ALLOCATION EFFICIENCY, the share of the bytes explore returned that belonged
  15. // to files the agent's final answer actually cited (see computeAllocation).
  16. //
  17. // `--envelope` additionally reports how the codegraph_explore responses were
  18. // DIVIDED across files — the per-file share of the source envelope (#1500).
  19. // `--answer <glob>` (repeatable, implies --envelope) marks the files that
  20. // actually answer the question and reports their combined share: bar 2 of the
  21. // CG-1/CG-22 allocation gate. See formatEnvelope for why it parses the
  22. // rendered markdown rather than the CG-4 diagnostic sidecar.
  23. //
  24. // ---------------------------------------------------------------------------
  25. // Why occupancy, and how it's measured
  26. // ---------------------------------------------------------------------------
  27. // A single-question A/B reports cost/tokens/time/tool-calls for ONE answer. It
  28. // cannot see what issue #1500 measured: a tool response stays in the window for
  29. // everything that follows, so it is charged against every later turn's headroom.
  30. // That is a per-session cost our single-question runs structurally miss.
  31. //
  32. // Tokens are MEASURED, not estimated at bytes/4. For assistant request k,
  33. // ctx_k = usage.input_tokens + cache_read_input_tokens + cache_creation_input_tokens
  34. // is the exact token count of that request's whole prompt. So
  35. // gap_k = ctx_k - ctx_{k-1}
  36. // is exactly the tokens appended since the previous request: the previous
  37. // assistant output (thinking + text + tool_use JSON) plus the tool_results and
  38. // user text that followed it. We split gap_k across those blocks in proportion
  39. // to their characters, which attributes each tool_result its measured share.
  40. // (Measured on real runs, explore output lands near 2.3 chars/token — bytes/4
  41. // under-counts it by ~40%, which is why the estimate isn't good enough.)
  42. //
  43. // Two traps this file works around, both verified against real logs:
  44. // * Claude Code emits ONE assistant event PER CONTENT BLOCK, all carrying the
  45. // same message.id and the same `usage`. Summing usage per event double-counts
  46. // every turn that emits both thinking and a tool_use — dedupe by message.id.
  47. // * The streamed `output_tokens` is a partial snapshot (observed `out=2` on a
  48. // turn that really generated ~1100). Never trust it; the char-proportional
  49. // split doesn't need it.
  50. //
  51. // Residual ≠ contributed. Content leaves the window two ways, and both are
  52. // tracked: a `compact_boundary` system event (everything prior is replaced by a
  53. // summary) and micro-compaction (ctx drops mid-run — oldest tool results are
  54. // dropped first, so eviction is applied FIFO).
  55. import { readFileSync } from 'fs';
  56. import { pathToFileURL } from 'url';
  57. // Nominal window for the share-of-window column. Override for a [1m] context.
  58. const WINDOW_TOKENS = Number(process.env.CG_WINDOW_TOKENS || 200_000);
  59. const CHARS_PER_TOKEN_FALLBACK = 3.0;
  60. /** Which tool family a tool_use belongs to. */
  61. function familyOf(name) {
  62. if (/codegraph/.test(name)) return 'codegraph';
  63. if (name === 'Read' || name === 'NotebookRead') return 'read';
  64. if (name === 'Grep' || name === 'Glob') return 'search';
  65. if (name === 'Bash' || name === 'BashOutput') return 'bash';
  66. return 'other';
  67. }
  68. const FAMILIES = ['codegraph', 'read', 'search', 'bash', 'other'];
  69. // The without-arm's way of getting the same bytes: reading and searching files.
  70. const FILE_ACCESS = ['read', 'search', 'bash'];
  71. // A Bash command that INVOKES the codegraph CLI, in any command position and by
  72. // any path. Mentions are not invocations: `grep codegraph src/`, `ls .codegraph`
  73. // and `which codegraph` all pass. Kept in step with run-all.sh's blocking hook.
  74. const CG_CLI_RE = /(^|[;&|(]|&&|\|\||\$\(|`)\s*(?:[A-Za-z_]\w*=\S*\s+)*[\w./~-]*codegraph(\s|$)/;
  75. const textOf = (content) =>
  76. Array.isArray(content) ? content.map((c) => c.text ?? (typeof c === 'string' ? c : JSON.stringify(c))).join('')
  77. : typeof content === 'string' ? content
  78. : content == null ? '' : JSON.stringify(content);
  79. /** Characters an assistant content block occupies once it is back in the prompt. */
  80. function assistantBlockChars(b) {
  81. if (b.type === 'text') return (b.text || '').length;
  82. if (b.type === 'thinking') return (b.thinking || '').length;
  83. if (b.type === 'tool_use') return JSON.stringify(b.input ?? {}).length + (b.name || '').length;
  84. return JSON.stringify(b).length;
  85. }
  86. /**
  87. * Parse one session (its segment files, in order) into tool + occupancy stats.
  88. * Exported so parse-bench-readme.mjs can aggregate without duplicating any of
  89. * this — deliberately NOT a separate module file: a new scripts/agent-eval/*.mjs
  90. * scores into the self-query eval fixture's own corpus and moves its numbers.
  91. */
  92. export function parseSession(files) {
  93. const events = [];
  94. for (const f of files) {
  95. for (const line of readFileSync(f, 'utf8').split('\n')) {
  96. if (!line) continue;
  97. try { events.push(JSON.parse(line)); } catch { /* partial line */ }
  98. }
  99. }
  100. const toolCalls = []; // display sequence
  101. const nameById = new Map(); // tool_use_id -> tool name
  102. const cliById = new Set(); // tool_use_ids that tried to run the codegraph CLI
  103. const counts = {}; // tool name -> calls
  104. // Attempts vs successes: run-all.sh's hook DENIES CLI invocations, and a
  105. // denied attempt puts no codegraph output in the window. Only a call that
  106. // actually returned content contaminates the arm.
  107. let initTools = null, result = null, raced = false, cliCalls = 0, cliContaminated = 0;
  108. const results = []; // one `result` event per session segment (multi-turn)
  109. let compactions = 0;
  110. // Raw codegraph_explore response text, in call order. Feeds the envelope view
  111. // (see formatEnvelope) — kept here rather than re-parsed from the log later so
  112. // a multi-segment session's responses stay in one ordered list.
  113. const exploreTexts = [];
  114. // A timeline of everything appended to the context, in order. `req` entries
  115. // are assistant requests (carrying that request's ctx); `add` entries are
  116. // characters appended (assistant output blocks, tool results, user text).
  117. const timeline = [];
  118. const seenMsgIds = new Set();
  119. for (const ev of events) {
  120. if (ev.type === 'system' && ev.subtype === 'init') {
  121. initTools = (ev.tools || []).filter((t) => /codegraph/.test(t));
  122. }
  123. if (ev.type === 'system' && (ev.subtype === 'compact_boundary' || ev.subtype === 'compaction')) {
  124. compactions++;
  125. timeline.push({ kind: 'compact' });
  126. }
  127. if (ev.type === 'assistant' && ev.message) {
  128. const id = ev.message.id;
  129. // One event per content block, same id + same usage: count usage once,
  130. // but take the content blocks from every event that carries the id.
  131. if (id && !seenMsgIds.has(id)) {
  132. seenMsgIds.add(id);
  133. const u = ev.message.usage || {};
  134. const ctx = (u.input_tokens || 0) + (u.cache_read_input_tokens || 0) + (u.cache_creation_input_tokens || 0);
  135. timeline.push({ kind: 'req', ctx, out: u.output_tokens || 0 });
  136. }
  137. for (const b of ev.message.content || []) {
  138. timeline.push({ kind: 'add', family: null, chars: assistantBlockChars(b) });
  139. if (b.type === 'tool_use') {
  140. nameById.set(b.id, b.name);
  141. counts[b.name] = (counts[b.name] || 0) + 1;
  142. let detail = '';
  143. if (b.name === 'Task') detail = ` [subagent_type=${b.input?.subagent_type ?? '?'}] ${(b.input?.description ?? '').slice(0, 40)}`;
  144. else if (/codegraph/.test(b.name)) detail = ` ${JSON.stringify(b.input?.query ?? b.input?.task ?? b.input?.symbol ?? '').slice(0, 60)}`;
  145. else if (b.name === 'Bash') {
  146. detail = ` ${(b.input?.command ?? '').slice(0, 50)}`;
  147. // An arm with no codegraph MCP can still shell out to the CLI — the
  148. // target repo carries the .codegraph/ index and the binary is on
  149. // PATH. That silently turns a "without" arm into codegraph-over-CLI.
  150. if (CG_CLI_RE.test(b.input?.command ?? '')) { cliCalls++; cliById.add(b.id); }
  151. }
  152. else if (b.name === 'Read') detail = ` ${(b.input?.file_path ?? '').split('/').slice(-1)[0]}`;
  153. toolCalls.push(`${b.name}${detail}`);
  154. }
  155. }
  156. }
  157. if (ev.type === 'user' && ev.message) {
  158. const content = ev.message.content;
  159. if (Array.isArray(content)) {
  160. for (const b of content) {
  161. if (b.type === 'tool_result') {
  162. const t = textOf(b.content);
  163. // MCP cold-start race: the agent fired before `serve --mcp` had
  164. // registered its tools, so it floundered into grep/Read. That
  165. // measures startup latency, not steady-state value — flag it.
  166. if (/No such tool available/.test(t)) raced = true;
  167. // A CLI attempt that came back an error was blocked (by the hook, or
  168. // by the binary being genuinely absent) and put nothing in context.
  169. if (cliById.has(b.tool_use_id) && !b.is_error) cliContaminated++;
  170. const name = nameById.get(b.tool_use_id) || '';
  171. if (/codegraph_explore/.test(name) && !b.is_error) exploreTexts.push(t);
  172. timeline.push({ kind: 'add', family: familyOf(name), chars: t.length, tool: name });
  173. } else {
  174. timeline.push({ kind: 'add', family: null, chars: textOf([b]).length });
  175. }
  176. }
  177. } else if (typeof content === 'string') {
  178. timeline.push({ kind: 'add', family: null, chars: content.length });
  179. }
  180. }
  181. if (ev.type === 'result') { result = ev; results.push(ev); }
  182. }
  183. // ---- Pass 1: chars/token, calibrated on tool-result-dominated gaps. ------
  184. // Splitting a gap in proportion to characters over-attributes to tool results
  185. // whenever the assistant's own output is under-represented in the transcript
  186. // (redacted/empty thinking blocks are the common case — a gap whose only
  187. // visible chars were a 73-char tool_result charged it the whole 830-token
  188. // delta, 5.5 tok/char). So calibrate the ratio on gaps that are ≥80% tool
  189. // result by characters, then price every result at that ratio.
  190. const reqIdx = timeline.map((t, i) => (t.kind === 'req' ? i : -1)).filter((i) => i >= 0);
  191. const gaps = [];
  192. for (let k = 1; k < reqIdx.length; k++) {
  193. const prev = timeline[reqIdx[k - 1]], cur = timeline[reqIdx[k]];
  194. let chars = 0, toolChars = 0, compacted = false;
  195. const byFamily = {};
  196. for (let i = reqIdx[k - 1] + 1; i < reqIdx[k]; i++) {
  197. const t = timeline[i];
  198. if (t.kind === 'compact') { compacted = true; continue; }
  199. if (t.kind !== 'add') continue;
  200. chars += t.chars;
  201. if (t.family) { toolChars += t.chars; byFamily[t.family] = (byFamily[t.family] || 0) + t.chars; }
  202. }
  203. gaps.push({ delta: cur.ctx - prev.ctx, chars, toolChars, byFamily, compacted });
  204. }
  205. const clean = gaps.filter((g) => !g.compacted && g.delta > 0 && g.chars > 500 && g.toolChars / g.chars >= 0.8);
  206. // A gap where the window also SHED content has a delta far below what was
  207. // added, which reads as absurdly dense text and would drag the whole run's
  208. // ratio with it. Shedding can only push a gap's chars/token UP, so take the
  209. // lower median as the honest centre and drop anything well above it, then
  210. // pool the survivors. (On runs that never shed, every ratio is within a few
  211. // percent of the others and this changes nothing.)
  212. const ratios = clean.map((g) => g.toolChars / g.delta).sort((a, b) => a - b);
  213. const lowerMedian = ratios.length ? ratios[Math.floor((ratios.length - 1) / 2)] : 0;
  214. let sumD = 0, sumC = 0;
  215. for (const g of clean) {
  216. if (lowerMedian > 0 && g.toolChars / g.delta > lowerMedian * 1.5) continue; // shed
  217. sumD += g.delta; sumC += g.toolChars;
  218. }
  219. if (sumD === 0) { // no clean gap — fall back to every growing gap, all chars
  220. for (const g of gaps) if (!g.compacted && g.delta > 0 && g.chars > 0) { sumD += g.delta; sumC += g.chars; }
  221. }
  222. const charsPerToken = sumD > 0 ? sumC / sumD : CHARS_PER_TOKEN_FALLBACK;
  223. const calibrated = sumD > 0;
  224. // How far a single result's token density strays from the run-level ratio.
  225. // On a gap that is almost entirely one tool result, `delta` IS that result's
  226. // token count, so |chars/ratio - delta| / delta is the attribution error for
  227. // that result. The median over such gaps is the metric's real error bar.
  228. const errs = [];
  229. for (const g of gaps) {
  230. if (g.compacted || g.delta <= 0 || g.chars <= 500) continue;
  231. if (g.toolChars / g.chars < 0.95) continue;
  232. errs.push(Math.abs(g.toolChars / charsPerToken - g.delta) / g.delta);
  233. }
  234. errs.sort((a, b) => a - b);
  235. const dispersion = errs.length ? errs[(errs.length - 1) >> 1] : null;
  236. // ---- Pass 2: attribute gap tokens, then apply evictions FIFO. ------------
  237. const contributed = Object.fromEntries(FAMILIES.map((f) => [f, 0]));
  238. const resultChars = Object.fromEntries(FAMILIES.map((f) => [f, 0]));
  239. const resultCount = Object.fromEntries(FAMILIES.map((f) => [f, 0]));
  240. for (const t of timeline) if (t.kind === 'add' && t.family) { resultChars[t.family] += t.chars; resultCount[t.family]++; }
  241. let queue = []; // resident contributions, oldest first
  242. let evicted = 0;
  243. const evict = (tokens) => {
  244. let left = tokens;
  245. while (left > 0 && queue.length) {
  246. const head = queue[0];
  247. if (head.tokens <= left) { left -= head.tokens; evicted += head.tokens; queue.shift(); }
  248. else { head.tokens -= left; evicted += left; left = 0; }
  249. }
  250. };
  251. for (const g of gaps) {
  252. if (g.compacted) {
  253. // Everything before the boundary is gone; the summary replaces it.
  254. evicted += queue.reduce((s, q) => s + q.tokens, 0);
  255. queue = [];
  256. }
  257. let toolTokens = 0;
  258. for (const [fam, ch] of Object.entries(g.byFamily)) {
  259. const tok = ch / charsPerToken;
  260. toolTokens += tok;
  261. contributed[fam] += tok;
  262. queue.push({ family: fam, tokens: tok });
  263. }
  264. // The gap grew by `delta`; the tool results account for `toolTokens` of it.
  265. // A shortfall means the window also shed content — micro-compaction drops
  266. // the OLDEST tool results first, so evict FIFO. The tolerance keeps
  267. // attribution noise (a run-level ratio priced against one gap's delta,
  268. // typically ±2%) from reading as an eviction; real shedding is thousands.
  269. const shortfall = toolTokens - g.delta;
  270. if (!g.compacted && shortfall > Math.max(200, toolTokens * 0.05)) evict(shortfall);
  271. }
  272. const residual = Object.fromEntries(FAMILIES.map((f) => [f, 0]));
  273. for (const q of queue) residual[q.family] += q.tokens;
  274. const ctxFinal = reqIdx.length ? timeline[reqIdx[reqIdx.length - 1]].ctx : 0;
  275. // The FIRST request's prompt is system + tool schemas + the question, before
  276. // any tool has answered. Differencing the arms' ctxBase prices codegraph's
  277. // FIXED occupancy — its tool schema and MCP `initialize` instructions — which
  278. // it pays whether or not the agent ever calls it.
  279. const ctxBase = reqIdx.length ? timeline[reqIdx[0]].ctx : 0;
  280. // Multi-turn: duration/cost/tokens are per-segment, so sum them. `result.usage`
  281. // is cumulative WITHIN a segment (verified: its in+cache+out equals the sum of
  282. // that segment's per-request prompts), so summing segments is correct and does
  283. // NOT double-count. It is a "tokens processed" figure — every request re-counts
  284. // the whole prefix — which is exactly why it can't answer the occupancy question.
  285. const sumUsage = (k) => results.reduce((s, r) => s + (r.usage?.[k] || 0), 0);
  286. const processed = sumUsage('input_tokens') + sumUsage('cache_read_input_tokens')
  287. + sumUsage('cache_creation_input_tokens') + sumUsage('output_tokens');
  288. return {
  289. files, toolCalls, counts, initTools, result, results, raced, cliCalls, cliContaminated,
  290. exploreTexts,
  291. // What the agent did after each explore — the free sufficiency signal (CG-8).
  292. sufficiency: classifySufficiency(events),
  293. // How much of what explore returned the answer actually drew on (CG-9).
  294. allocation: computeAllocation(exploreTexts, finalAnswerText(events)),
  295. ok: results.length > 0 && results.every((r) => r.subtype === 'success'),
  296. turns: reqIdx.length,
  297. tools: toolCalls.filter((t) => !t.startsWith('ToolSearch')).length,
  298. reads: counts.Read || 0,
  299. grep: (counts.Grep || 0) + (counts.Glob || 0),
  300. cg: Object.entries(counts).filter(([n]) => /codegraph/.test(n)).reduce((s, [, v]) => s + v, 0),
  301. dur: results.reduce((s, r) => s + (r.duration_ms || 0), 0) / 1000,
  302. cost: results.reduce((s, r) => s + (r.total_cost_usd || 0), 0),
  303. processed,
  304. occupancy: {
  305. ctxFinal, ctxBase, windowTokens: WINDOW_TOKENS,
  306. charsPerToken, calibrated, compactions, dispersion, evicted: Math.round(evicted),
  307. residual: Object.fromEntries(FAMILIES.map((f) => [f, Math.round(residual[f])])),
  308. contributed: Object.fromEntries(FAMILIES.map((f) => [f, Math.round(contributed[f])])),
  309. chars: resultChars, results: resultCount,
  310. residualFileAccess: Math.round(FILE_ACCESS.reduce((s, f) => s + residual[f], 0)),
  311. contributedFileAccess: Math.round(FILE_ACCESS.reduce((s, f) => s + contributed[f], 0)),
  312. charsFileAccess: FILE_ACCESS.reduce((s, f) => s + resultChars[f], 0),
  313. },
  314. };
  315. }
  316. /** The occupancy block, as printed under a run and reused by the aggregator. */
  317. export function formatOccupancy(s, indent = ' ') {
  318. const o = s.occupancy;
  319. const n = (x) => x.toLocaleString('en-US');
  320. const pctCtx = (t) => (o.ctxFinal > 0 ? ((t / o.ctxFinal) * 100).toFixed(1) : '0.0');
  321. const pctWin = (t) => ((t / o.windowTokens) * 100).toFixed(1);
  322. const rows = [];
  323. const row = (label, tok, chars, results) => rows.push(
  324. `${indent} ${label.padEnd(18)}${(n(tok) + ' tok').padStart(12)} ${(pctCtx(tok) + '%').padStart(6)} of ctx ` +
  325. `${(pctWin(tok) + '%').padStart(6)} of ${Math.round(o.windowTokens / 1000)}k win` +
  326. (chars !== undefined ? ` (${n(chars)} chars, ${results} result${results === 1 ? '' : 's'})` : '')
  327. );
  328. const out = [`${indent}Residual context occupancy at end of run:`];
  329. out.push(`${indent} ${'final context'.padEnd(18)}${(n(o.ctxFinal) + ' tok').padStart(12)} ${(pctWin(o.ctxFinal) + '%').padStart(6)} of ${Math.round(o.windowTokens / 1000)}k window`);
  330. row('codegraph', o.residual.codegraph, o.chars.codegraph, o.results.codegraph);
  331. row('Read', o.residual.read, o.chars.read, o.results.read);
  332. row('Grep/Glob', o.residual.search, o.chars.search, o.results.search);
  333. row('Bash', o.residual.bash, o.chars.bash, o.results.bash);
  334. row('→ file-access', o.residualFileAccess, o.charsFileAccess,
  335. o.results.read + o.results.search + o.results.bash);
  336. row('other tools', o.residual.other, o.chars.other, o.results.other);
  337. const toolTotal = Object.values(o.residual).reduce((a, b) => a + b, 0);
  338. row('base (prompt+prose)', Math.max(0, o.ctxFinal - toolTotal));
  339. out.push(`${indent} ${' of which fixed'.padEnd(18)}${(n(o.ctxBase) + ' tok').padStart(12)} system + tool schemas + question, before any tool answered`);
  340. out.push(...rows);
  341. const dropped = o.contributed.codegraph + o.contributedFileAccess + o.contributed.other
  342. - (o.residual.codegraph + o.residualFileAccess + o.residual.other);
  343. out.push(
  344. `${indent} measure: ${o.charsPerToken.toFixed(2)} chars/tok ${o.calibrated ? 'measured' : '(FALLBACK — no clean gap to calibrate on)'}` +
  345. (o.dispersion !== null ? ` ±${(o.dispersion * 100).toFixed(1)}%` : '') +
  346. ` · turns ${s.turns} · compactions ${o.compactions}` +
  347. (o.evicted > 0 || dropped > 1 ? ` · evicted ${n(o.evicted)} tok` : '')
  348. );
  349. return out.join('\n');
  350. }
  351. /**
  352. * One codegraph_explore response, split into the per-file source sections the
  353. * allocator divided its budget across.
  354. *
  355. * Parsed out of the RENDERED MARKDOWN, not the CG-4 diagnostic sidecar: the
  356. * sidecar only exists on a post-CG-4 build, so it cannot measure a baseline arm.
  357. * The markdown parse is the only instrument that measures both arms of a
  358. * new-vs-baseline A/B the same way. Both the envelope view and the allocation
  359. * metric key off this one parse.
  360. *
  361. * Returns `{ envelope, files: [{ path, chars, symbols }] }`, where `symbols` are
  362. * the names the section header lists — filtered to the ones the file DEFINES.
  363. * The header renders `name(kind)` for every node in the shipped clusters, and
  364. * that includes edge-kind pseudo-entries for call sites (`mutateElement(calls)`
  365. * on a file that merely calls it). Attributing a file from those would credit
  366. * every caller of a cited symbol — measured on a real excalidraw run, it marked
  367. * `dragElements.ts` used because the answer named `mutateElement`.
  368. */
  369. /** NodeKind values that mean "this file defines it" (src/types.ts, less file/import/export). */
  370. const DEFINING_KINDS = new Set([
  371. 'module', 'class', 'struct', 'interface', 'trait', 'protocol', 'function', 'method',
  372. 'property', 'field', 'variable', 'constant', 'enum', 'enum_member', 'type_alias',
  373. 'namespace', 'route', 'component',
  374. ]);
  375. export function parseExploreCall(text) {
  376. const src = String(text ?? '');
  377. const re = /^\*\*`([^`]+)`\*\*(.*)$/gm;
  378. const marks = [];
  379. let m;
  380. while ((m = re.exec(src)) !== null) marks.push({ path: m[1], header: m[2] || '', at: m.index });
  381. if (!marks.length) return { envelope: 0, files: [] };
  382. // Sections run to the next header, or to the trailing guidance quote.
  383. const tail = src.indexOf('\n> ', marks[marks.length - 1].at);
  384. const end = tail === -1 ? src.length : tail;
  385. const files = marks.map((mark, i) => ({
  386. path: mark.path,
  387. chars: (i + 1 < marks.length ? marks[i + 1].at : end) - mark.at,
  388. // `mutateElement(calls), onFinished(variable), +32 more` → the defined ones.
  389. symbols: [...mark.header.matchAll(/([A-Za-z_$][\w$.]*)\(([a-z_]+)\)/g)]
  390. .filter((s) => DEFINING_KINDS.has(s[2]))
  391. .map((s) => ({ name: s[1], kind: s[2] })),
  392. }));
  393. return { envelope: files.reduce((s, f) => s + f.chars, 0), files };
  394. }
  395. /**
  396. * How the codegraph_explore responses the agent received were DIVIDED across
  397. * files — the per-file share of the source envelope (#1500 / epic CG-1).
  398. *
  399. * `answerGlobs` marks the files that actually answer the question; the summary
  400. * reports their combined share, which is bar 2 of the CG-1/CG-22 gate. That is
  401. * HAND-SPECIFIED ground truth; the allocation metric below infers the same
  402. * intersection from the agent's own answer instead.
  403. */
  404. export function formatEnvelope(exploreTexts, answerGlobs = [], indent = ' ') {
  405. // `tools/cache/**` -> /^tools\/cache\/.*$/ . Same semantics as probe-allocation.
  406. // The `**` sentinel is written as an escape, never a literal NUL byte — a raw
  407. // one makes git treat this whole script as binary and costs every future diff.
  408. const glob2re = (glob) => {
  409. const S = '\\u0000';
  410. const body = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&')
  411. .replace(/\*\*/g, S).replace(/\*/g, '[^/]*').replaceAll(S, '.*');
  412. return new RegExp(`^${body}$`);
  413. };
  414. const answerRes = answerGlobs.map(glob2re);
  415. const isAnswer = (p) => answerRes.some((re) => re.test(p));
  416. // Share is over the sum of the per-file sections, i.e. of the source envelope
  417. // the allocator divides.
  418. const pooled = new Map();
  419. let envelope = 0;
  420. for (const text of exploreTexts) {
  421. for (const f of parseExploreCall(text).files) {
  422. pooled.set(f.path, (pooled.get(f.path) ?? 0) + f.chars);
  423. envelope += f.chars;
  424. }
  425. }
  426. const ranked = [...pooled.entries()]
  427. .map(([path, chars]) => ({ path, chars, share: envelope ? chars / envelope : 0, answer: isAnswer(path) }))
  428. .sort((a, b) => b.chars - a.chars);
  429. const answerChars = ranked.filter((r) => r.answer).reduce((s, r) => s + r.chars, 0);
  430. const pct = (f) => `${(f * 100).toFixed(1)}%`;
  431. const out = [];
  432. out.push(`${indent}Explore envelope: ${envelope.toLocaleString('en-US')} chars over ${exploreTexts.length} response(s)`);
  433. if (answerGlobs.length) {
  434. out.push(`${indent} answer-set share: ${pct(envelope ? answerChars / envelope : 0)} | top file answers: ${ranked[0]?.answer ?? false}`);
  435. }
  436. for (const f of ranked.slice(0, 12)) {
  437. out.push(`${indent} ${f.answer ? '*' : ' '} ${pct(f.share).padStart(6)} ${String(f.chars).padStart(6)} ${f.path}`);
  438. }
  439. if (ranked.length > 12) out.push(`${indent} … ${ranked.length - 12} more files`);
  440. return out.join('\n');
  441. }
  442. // ---------------------------------------------------------------------------
  443. // Explore allocation efficiency (CG-9)
  444. // ---------------------------------------------------------------------------
  445. // The envelope view above needs a human to say which files answer the question
  446. // (`--answer <glob>`). This one reads that off the AGENT'S OWN FINAL ANSWER and
  447. // reports a single number every run:
  448. //
  449. // allocation efficiency = bytes explore returned for files the answer drew on
  450. // ─────────────────────────────────────────────────
  451. // all bytes explore returned
  452. //
  453. // That is the #1500 defect as a number. On the CG-1 baseline self-query it sat
  454. // around 16–30%: one correct file, one marginal, three pure noise.
  455. //
  456. // ATTRIBUTION IS IMPERFECT AND THE ERROR IS ONE-SIDED. An agent can use a file's
  457. // source to rule it out, or to build a mental model, and never cite it — that
  458. // reads as waste. So this is a RELATIVE metric: valid for comparing two builds
  459. // on the SAME question, not as an absolute claim about how much of an envelope
  460. // earned its place. Quoting it as an absolute efficiency figure is a misuse.
  461. //
  462. // Two citation channels, deliberately ranked so the weaker one is separable:
  463. // * PATH — the answer names the file (`lib/response.js:126-220`, or the bare
  464. // `utils.js:225` agents drop into prose). Strong, and reported on its own as
  465. // the conservative floor.
  466. // * SYMBOL — the answer cites a symbol in a code span and only that file's
  467. // section header lists it. Catches answers written entirely in symbol names.
  468. // Guarded: a name carried by ≥3 of the returned files is too generic to
  469. // attribute and is dropped, or `send`/`get` would mark half the envelope
  470. // used and bias the metric optimistic — the one direction it must not lean.
  471. /** Symbol tokens too common in prose/code spans to attribute a file from. */
  472. const SYMBOL_STOPWORDS = new Set([
  473. 'function', 'return', 'const', 'this', 'true', 'false', 'null', 'void', 'undefined',
  474. 'string', 'object', 'number', 'boolean', 'array', 'class', 'import', 'export',
  475. 'async', 'await', 'default', 'type', 'value', 'name', 'data', 'self', 'super',
  476. 'else', 'then', 'case', 'from', 'with', 'when', 'where', 'that', 'they', 'this',
  477. ]);
  478. /** A name on this many of the returned files stops identifying any one of them. */
  479. const SYMBOL_AMBIGUITY_LIMIT = 3;
  480. /** Below this length a token is far more likely prose than a symbol citation. */
  481. const MIN_SYMBOL_LEN = 4;
  482. /**
  483. * What the agent's final answer CITES: file paths, and the identifiers it puts
  484. * in code spans. `extensions` are the file extensions actually present in the
  485. * envelope — the gate that lets `utils.js:225` through as a file citation while
  486. * rejecting `res.send` and `mime.contentType`, which are the same token shape.
  487. */
  488. export function answerCitations(text, extensions = new Set()) {
  489. const src = String(text ?? '');
  490. const paths = new Set();
  491. // Dotted path with at least one directory: `lib/response.js:126-220`.
  492. for (const m of src.matchAll(/(?:[\w@.+-]+\/)+[\w@.+-]+\.[A-Za-z]\w*/g)) paths.add(m[0]);
  493. // Bare basename, only when its extension is one the envelope actually shipped.
  494. for (const m of src.matchAll(/\b[\w@+-]+\.[A-Za-z]\w*/g)) {
  495. const ext = m[0].slice(m[0].lastIndexOf('.') + 1).toLowerCase();
  496. if (extensions.has(ext)) paths.add(m[0]);
  497. }
  498. // Identifiers inside code spans. Prose mentions are excluded on purpose: a
  499. // backtick is the agent marking the token as code, which is the whole signal.
  500. const symbols = new Set();
  501. for (const span of src.matchAll(/`([^`\n]+)`/g)) {
  502. for (const t of span[1].matchAll(/[A-Za-z_$][\w$]*/g)) {
  503. const tok = t[0];
  504. if (tok.length >= MIN_SYMBOL_LEN && !SYMBOL_STOPWORDS.has(tok.toLowerCase())) symbols.add(tok);
  505. }
  506. }
  507. return { paths: [...paths], symbols };
  508. }
  509. /**
  510. * Allocation efficiency over one session's explore responses and its final
  511. * answer(s). Per call AND pooled over the run — a run-level number alone hides
  512. * the common shape where call 1 is on target and call 3 is pure noise.
  513. */
  514. export function computeAllocation(exploreTexts, answerText) {
  515. const calls = exploreTexts.map(parseExploreCall).filter((c) => c.files.length);
  516. const extensions = new Set();
  517. for (const c of calls) {
  518. for (const f of c.files) {
  519. const dot = f.path.lastIndexOf('.');
  520. if (dot > 0) extensions.add(f.path.slice(dot + 1).toLowerCase());
  521. }
  522. }
  523. const cited = answerCitations(answerText, extensions);
  524. // symbol -> the returned files whose header lists it as DEFINED. A `variable`
  525. // is usually an import binding (`var compileETag = require('./utils')…`), so
  526. // when the same name is also a real definition somewhere in the envelope, the
  527. // definition wins and the aliasing file is not credited.
  528. const symbolFiles = new Map();
  529. for (const c of calls) {
  530. for (const f of c.files) {
  531. for (const s of f.symbols) {
  532. if (!symbolFiles.has(s.name)) symbolFiles.set(s.name, { strong: new Set(), weak: new Set() });
  533. const owners = symbolFiles.get(s.name);
  534. (s.kind === 'variable' || s.kind === 'constant' ? owners.weak : owners.strong).add(f.path);
  535. }
  536. }
  537. }
  538. const ownersOf = (name) => {
  539. const o = symbolFiles.get(name);
  540. return o ? (o.strong.size ? o.strong : o.weak) : null;
  541. };
  542. // `lib/response.js` is cited by `lib/response.js`, by `response.js`, and by an
  543. // absolute path ending in it — samePath already encodes exactly that.
  544. const viaPath = (path) => cited.paths.some((p) => samePath(path, p));
  545. const viaSymbol = (path) => {
  546. for (const s of cited.symbols) {
  547. const owners = ownersOf(s);
  548. if (owners && owners.size < SYMBOL_AMBIGUITY_LIMIT && owners.has(path)) return s;
  549. }
  550. return null;
  551. };
  552. const verdict = new Map(); // path -> { via, symbol }
  553. const judge = (path) => {
  554. if (!verdict.has(path)) {
  555. if (viaPath(path)) verdict.set(path, { via: 'path' });
  556. else {
  557. const s = viaSymbol(path);
  558. verdict.set(path, s ? { via: 'symbol', symbol: s } : { via: null });
  559. }
  560. }
  561. return verdict.get(path);
  562. };
  563. const perCall = calls.map((c, i) => {
  564. const files = c.files.map((f) => ({ ...f, ...judge(f.path) }));
  565. const used = files.filter((f) => f.via).reduce((s, f) => s + f.chars, 0);
  566. const usedPath = files.filter((f) => f.via === 'path').reduce((s, f) => s + f.chars, 0);
  567. return {
  568. call: i + 1, envelope: c.envelope, used, usedPath, files,
  569. efficiency: c.envelope ? used / c.envelope : 0,
  570. };
  571. });
  572. // Pooled: a file returned twice is charged twice, because it occupied the
  573. // window twice. Same accounting as formatEnvelope.
  574. const pooled = new Map();
  575. let envelope = 0;
  576. for (const c of perCall) {
  577. for (const f of c.files) {
  578. const prev = pooled.get(f.path) ?? { path: f.path, chars: 0, calls: 0, via: f.via, symbol: f.symbol };
  579. prev.chars += f.chars; prev.calls++;
  580. pooled.set(f.path, prev);
  581. envelope += f.chars;
  582. }
  583. }
  584. const files = [...pooled.values()].sort((a, b) => b.chars - a.chars);
  585. const used = files.filter((f) => f.via).reduce((s, f) => s + f.chars, 0);
  586. const usedPath = files.filter((f) => f.via === 'path').reduce((s, f) => s + f.chars, 0);
  587. return {
  588. calls: perCall, files, envelope, used, usedPath,
  589. efficiency: envelope ? used / envelope : 0,
  590. efficiencyPath: envelope ? usedPath / envelope : 0,
  591. filesReturned: files.length,
  592. filesUsed: files.filter((f) => f.via).length,
  593. hasAnswer: String(answerText ?? '').trim().length > 0,
  594. };
  595. }
  596. /**
  597. * Every answered codegraph_explore response in a transcript, in call order.
  598. * parseSession collects these as it walks the timeline; this is the same list
  599. * for callers that only have the raw events (parse-session.mjs).
  600. */
  601. export function collectExploreTexts(events) {
  602. const nameById = new Map();
  603. const texts = [];
  604. for (const ev of events) {
  605. const content = ev?.message?.content;
  606. if (!Array.isArray(content)) continue;
  607. for (const b of content) {
  608. if (b.type === 'tool_use') nameById.set(b.id, b.name);
  609. else if (b.type === 'tool_result' && !b.is_error
  610. && /codegraph_explore/.test(nameById.get(b.tool_use_id) || '')) texts.push(textOf(b.content));
  611. }
  612. }
  613. return texts;
  614. }
  615. /**
  616. * The final answer a session produced. Headless stream-json carries it on the
  617. * `result` event (one per resumed segment, all of which are answers); an
  618. * interactive transcript has none, so fall back to the LAST assistant text of
  619. * the main thread — intermediate narration would flood the citation set.
  620. */
  621. export function finalAnswerText(events) {
  622. const answers = events.filter((e) => e.type === 'result' && typeof e.result === 'string').map((e) => e.result);
  623. if (answers.length) return answers.join('\n\n');
  624. let last = '';
  625. for (const ev of events) {
  626. if (ev.type !== 'assistant' || (ev.parent_tool_use_id ?? null) !== null) continue;
  627. const text = (ev.message?.content || []).filter((b) => b.type === 'text').map((b) => b.text || '').join('');
  628. if (text.trim()) last = text;
  629. }
  630. return last;
  631. }
  632. /** The allocation block, as printed under a run and reused by aggregators. */
  633. export function formatAllocation(s, indent = ' ') {
  634. const a = s.allocation;
  635. if (!a || !a.calls.length) return `${indent}Explore allocation: no codegraph_explore responses with source sections`;
  636. const pct = (f) => `${(f * 100).toFixed(1)}%`;
  637. const n = (x) => x.toLocaleString('en-US');
  638. const out = [
  639. `${indent}Explore allocation — share of returned bytes the answer used ` +
  640. `(${a.calls.length} call${a.calls.length === 1 ? '' : 's'}, ${a.filesReturned} file${a.filesReturned === 1 ? '' : 's'}):`,
  641. `${indent} efficiency ${pct(a.efficiency).padStart(6)} ${n(a.used)} of ${n(a.envelope)} chars` +
  642. ` (${a.filesUsed}/${a.filesReturned} files cited; path-cited alone ${pct(a.efficiencyPath)})`,
  643. ];
  644. if (!a.hasAnswer) out.push(`${indent} !! no final answer text found — efficiency is not meaningful for this run`);
  645. for (const c of a.calls) {
  646. out.push(`${indent} call ${c.call}: ${pct(c.efficiency).padStart(6)} ${n(c.used)}/${n(c.envelope)} chars` +
  647. ` ${c.files.filter((f) => f.via).length}/${c.files.length} files`);
  648. }
  649. for (const f of a.files.slice(0, 12)) {
  650. const share = a.envelope ? f.chars / a.envelope : 0;
  651. const why = f.via === 'symbol' ? `symbol \`${f.symbol}\`` : f.via === 'path' ? 'path' : '—';
  652. out.push(`${indent} ${f.via ? '*' : ' '} ${pct(share).padStart(6)} ${String(f.chars).padStart(6)} ${f.path} ${why}`);
  653. }
  654. if (a.files.length > 12) out.push(`${indent} … ${a.files.length - 12} more files`);
  655. out.push(`${indent} note: relative metric — attribution is by citation, so compare builds on the same question, not absolutes.`);
  656. return out.join('\n');
  657. }
  658. // ---------------------------------------------------------------------------
  659. // Explore sufficiency (CG-8)
  660. // ---------------------------------------------------------------------------
  661. // The agent's NEXT action after a codegraph_explore is free ground truth about
  662. // whether that response was enough. The buckets are chosen so each one maps to
  663. // a distinct fix:
  664. //
  665. // another codegraph call insufficient — the response did not answer
  666. // Read of a file we RETURNED allocation bug — right file, wrong bytes
  667. // Read of a file we did NOT recall bug — the file never surfaced
  668. // Grep/Glob recall bug (weaker: the agent is still hunting)
  669. // anything else / no tool sufficient — the agent moved on
  670. //
  671. // Three rules keep this honest:
  672. // * Only a call issued in a LATER assistant message counts as a reaction. A
  673. // Read fired in the same message as the explore was issued before its
  674. // response existed, so it cannot be a verdict on it (those are counted
  675. // separately as `concurrent`).
  676. // * ToolSearch/TodoWrite are stepped over: loading a deferred tool schema or
  677. // ticking a checklist says nothing about the response.
  678. // * SUBAGENT CALLS ARE A SEPARATE THREAD. Claude Code interleaves a subagent's
  679. // tool calls into the same stream, tagged `parent_tool_use_id` — verified on
  680. // a real run where a delegated search's greps landed between the parent's
  681. // own calls. Reactions are matched within one thread, or the subagent's
  682. // first grep would be scored as the parent's verdict on an explore it never
  683. // saw.
  684. //
  685. // A delegation (`Agent`/`Task`) is judged by what the SUBAGENT did first, since
  686. // that thread is right there in the transcript. Scoring the delegation itself as
  687. // "moved on" would have called this run sufficient while the subagent was off
  688. // grepping for the file — the one direction of error a tuning metric must not
  689. // have. A delegation that never runs a tool stays "moved on".
  690. /** Tools that carry no signal about whether the previous response was enough. */
  691. const TRANSPARENT_TOOLS = new Set(['ToolSearch', 'TodoWrite']);
  692. /** Tools that hand the work to a subagent whose thread we then judge instead. */
  693. const DELEGATION_TOOLS = new Set(['Agent', 'Task']);
  694. /** Buckets, worst → best. Labels double as the summary rows. */
  695. const SUFFICIENCY = [
  696. ['explore_again', 'explore again', 'insufficient: did not answer'],
  697. ['read_returned', 'Read a file we returned', 'allocation: right file, wrong bytes'],
  698. ['read_missed', 'Read a file we did not return', 'recall: file never surfaced'],
  699. ['search', 'Grep/Glob', 'recall (weak): still hunting for the file'],
  700. ['sufficient', 'moved on / answered', 'sufficient'],
  701. ];
  702. const SUFFICIENCY_KEYS = SUFFICIENCY.map(([k]) => k);
  703. // Shell equivalents of Read and of Grep. Both arms have Bash, and on small
  704. // repos an agent reaches for `sed -n 100,200p file` as readily as for Read —
  705. // counting only the Read tool would score those explores as sufficient.
  706. const BASH_READ_RE = /(?:^|[;&|]|\$\(|`)\s*(?:sudo\s+)?(?:cat|bat|head|tail|less|more|nl|sed|awk)\s+([^\n|;&]*)/;
  707. const BASH_SEARCH_RE = /(?:^|[;&|]|\$\(|`)\s*(?:sudo\s+)?(?:grep|egrep|fgrep|rg|ag|ack|find|fd|ls|tree)\b/;
  708. /** What a Bash command is really doing, as far as retrieval is concerned. */
  709. function bashIntent(cmd) {
  710. const c = String(cmd || '');
  711. // A heredoc or a redirect is WRITING a file — `cat <<EOF > x` must not read
  712. // as a Read.
  713. if (!/<</.test(c) && !/>\s*\S/.test(c)) {
  714. const m = BASH_READ_RE.exec(c);
  715. if (m) {
  716. // Drop flags and numeric arguments (`sed -n '100,200p' lib/x.js`), then
  717. // take the last path-shaped token.
  718. const args = m[1].split(/\s+/).filter((a) => a && !a.startsWith('-') && !/^['"]?\d/.test(a));
  719. const path = args.reverse().find((a) => /[/.]/.test(a));
  720. if (path) return { kind: 'read', path: path.replace(/^['"]|['"]$/g, '') };
  721. }
  722. }
  723. if (BASH_SEARCH_RE.test(c)) return { kind: 'search' };
  724. return null;
  725. }
  726. const normPath = (p) => String(p ?? '').replace(/\\/g, '/').replace(/^\.\//, '');
  727. /** Same file, with either side repo-relative and the other absolute. */
  728. function samePath(a, b) {
  729. const x = normPath(a), y = normPath(b);
  730. if (!x || !y) return false;
  731. return x === y || x.endsWith('/' + y) || y.endsWith('/' + x);
  732. }
  733. /**
  734. * The files whose SOURCE an explore response returned — its per-file sections,
  735. * which start with the unique ``**` `` marker (FILE_SECTION_PREFIX in
  736. * src/mcp/tools.ts). formatEnvelope keys off the same marker; it needs the byte
  737. * offsets too, which is why it re-scans rather than calling this.
  738. */
  739. export function exploreReturnedFiles(text) {
  740. return [...String(text ?? '').matchAll(/^\*\*`([^`]+)`\*\*/gm)].map((m) => m[1]);
  741. }
  742. // Every path-shaped token anywhere in a response — flow steps, blast radius,
  743. // symbol lists. A file in here but NOT in the returned set was POINTED AT and
  744. // not delivered, which is a different (and more damning) miss than one the
  745. // response never mentioned at all.
  746. const PATH_TOKEN_RE = /(?:[\w@.+-]+\/)+[\w@.+-]+\.[A-Za-z]\w*/g;
  747. /**
  748. * The reaction one action represents, given what the explore had returned.
  749. * `earlier` is what PREVIOUS explores in the same thread returned: a re-read of
  750. * a file we already shipped is an allocation miss wherever it was shipped, and
  751. * filing it as recall would point the fix at the wrong end of the pipeline.
  752. */
  753. function reactionOf(action, returned, mentioned, earlier = []) {
  754. const { name, input } = action;
  755. const readOf = (path, prefix) => {
  756. const base = normPath(path).split('/').pop() || String(path ?? '');
  757. if (returned.some((r) => samePath(path, r))) return { bucket: 'read_returned', next: `${prefix}Read ${base}` };
  758. if (earlier.some((r) => samePath(path, r))) return { bucket: 'read_returned', next: `${prefix}Read ${base} (returned by an earlier explore)` };
  759. const named = mentioned.some((m) => samePath(path, m));
  760. return { bucket: 'read_missed', next: `${prefix}Read ${base}${named ? ' (named, not returned)' : ''}`, named };
  761. };
  762. if (/codegraph/.test(name)) return { bucket: 'explore_again', next: name.replace(/^mcp__[^_]*__/, '') };
  763. if (name === 'Read' || name === 'NotebookRead') return readOf(input.file_path ?? input.notebook_path, '');
  764. if (name === 'Grep' || name === 'Glob') return { bucket: 'search', next: name };
  765. if (name === 'Bash') {
  766. const intent = bashIntent(input.command);
  767. if (intent?.kind === 'read') return readOf(intent.path, 'Bash ');
  768. if (intent?.kind === 'search') return { bucket: 'search', next: 'Bash search' };
  769. }
  770. return { bucket: 'sufficient', next: name };
  771. }
  772. /** Is this action one of the ways an agent gets file bytes into its head? */
  773. const isFileAccess = (a) =>
  774. a.name === 'Read' || a.name === 'NotebookRead' || a.name === 'Grep' || a.name === 'Glob'
  775. || (a.name === 'Bash' && bashIntent(a.input?.command) !== null);
  776. /**
  777. * Bucket every answered codegraph_explore call in a transcript by what the
  778. * agent did next. Takes the raw JSONL events so it serves both transcript
  779. * shapes: stream-json runs (parse-run.mjs) and interactive session logs
  780. * (parse-session.mjs) — both emit one assistant event per content block with
  781. * `message.id`, and tool results as `tool_result` blocks in user messages.
  782. */
  783. export function classifySufficiency(events) {
  784. // One action list PER THREAD: 'main', plus one per subagent (keyed by the
  785. // delegating tool_use id, which is what `parent_tool_use_id` carries).
  786. const threads = new Map();
  787. const nameById = new Map();
  788. const textById = new Map(); // explore tool_use_id -> response text
  789. for (const ev of events) {
  790. const content = ev?.message?.content;
  791. if (!Array.isArray(content)) continue;
  792. const thread = ev.parent_tool_use_id ?? 'main';
  793. if (ev.type === 'assistant') {
  794. if (!threads.has(thread)) threads.set(thread, []);
  795. const list = threads.get(thread);
  796. for (const b of content) {
  797. if (b.type !== 'tool_use') continue;
  798. nameById.set(b.id, b.name);
  799. // No message.id (never seen on a real log) degrades to "every call is
  800. // its own message", i.e. same-message calls read as reactions.
  801. list.push({ msgId: ev.message.id || `#${thread}-${list.length}`, id: b.id, name: b.name, input: b.input || {} });
  802. }
  803. } else if (ev.type === 'user') {
  804. for (const b of content) {
  805. if (b.type !== 'tool_result') continue;
  806. const name = nameById.get(b.tool_use_id) || '';
  807. if (/codegraph_explore/.test(name) && !b.is_error) textById.set(b.tool_use_id, textOf(b.content));
  808. }
  809. }
  810. }
  811. const calls = [];
  812. let errors = 0, concurrent = 0;
  813. // What a delegation really did: the subagent's first substantive call. A
  814. // nested delegation is skipped rather than followed, so a subagent that only
  815. // spawns another subagent leaves the call as "moved on".
  816. const throughDelegation = (action, returned, mentioned, earlier) => {
  817. const first = (threads.get(action.id) || []).find((x) => !TRANSPARENT_TOOLS.has(x.name) && !DELEGATION_TOOLS.has(x.name));
  818. if (!first) return { bucket: 'sufficient', next: action.name };
  819. const r = reactionOf(first, returned, mentioned, earlier);
  820. return { ...r, next: `${action.name} → ${r.next}` };
  821. };
  822. for (const [thread, actions] of threads) {
  823. const earlier = []; // files previous explores in THIS thread already shipped
  824. for (let i = 0; i < actions.length; i++) {
  825. const a = actions[i];
  826. if (!/codegraph_explore/.test(a.name)) continue;
  827. const text = textById.get(a.id);
  828. // No response text = the call errored, or the run ended before it
  829. // returned. Nothing to judge the sufficiency of; count it and move on.
  830. if (text === undefined) { errors++; continue; }
  831. const returned = exploreReturnedFiles(text);
  832. const mentioned = text.match(PATH_TOKEN_RE) || [];
  833. let reaction = { bucket: 'sufficient', next: '(final answer)' };
  834. for (let j = i + 1; j < actions.length; j++) {
  835. const b = actions[j];
  836. if (b.msgId === a.msgId) { if (isFileAccess(b)) concurrent++; continue; }
  837. if (TRANSPARENT_TOOLS.has(b.name)) continue;
  838. reaction = DELEGATION_TOOLS.has(b.name)
  839. ? throughDelegation(b, returned, mentioned, earlier)
  840. : reactionOf(b, returned, mentioned, earlier);
  841. break;
  842. }
  843. earlier.push(...returned);
  844. calls.push({ thread, query: String(a.input.query ?? ''), files: returned.length, chars: text.length, ...reaction });
  845. }
  846. }
  847. const counts = Object.fromEntries(SUFFICIENCY_KEYS.map((k) => [k, 0]));
  848. for (const c of calls) counts[c.bucket]++;
  849. return { calls, counts, errors, concurrent, answered: calls.length };
  850. }
  851. /** The sufficiency block, as printed under a run and reused by aggregators. */
  852. export function formatSufficiency(s, indent = ' ') {
  853. const f = s.sufficiency;
  854. if (!f.answered) {
  855. return `${indent}Explore sufficiency: no answered codegraph_explore calls`
  856. + (f.errors ? ` (${f.errors} errored or never returned)` : '');
  857. }
  858. const pct = (n) => ((n / f.answered) * 100).toFixed(0) + '%';
  859. const out = [`${indent}Explore sufficiency — what the agent did NEXT (${f.answered} answered call${f.answered === 1 ? '' : 's'}):`];
  860. for (const [key, label, meaning] of SUFFICIENCY) {
  861. out.push(`${indent} ${String(f.counts[key]).padStart(3)} ${pct(f.counts[key]).padStart(4)} ${label.padEnd(31)}${meaning}`);
  862. }
  863. f.calls.forEach((c, i) => {
  864. const q = c.query.length > 46 ? c.query.slice(0, 45) + '…' : c.query;
  865. const where = c.thread && c.thread !== 'main' ? ' [subagent]' : '';
  866. out.push(`${indent} ${i + 1}.${where} "${q}" [${c.files} file${c.files === 1 ? '' : 's'}] → ${c.next}`);
  867. });
  868. const notes = [];
  869. if (f.errors) notes.push(`${f.errors} errored/unanswered call${f.errors === 1 ? '' : 's'} (not bucketed)`);
  870. if (f.concurrent) notes.push(`${f.concurrent} file-access call${f.concurrent === 1 ? '' : 's'} in the SAME message as an explore (not a reaction)`);
  871. if (notes.length) out.push(`${indent} note: ${notes.join(' · ')}`);
  872. return out.join('\n');
  873. }
  874. // ---------------------------------------------------------------------------
  875. // `--selftest`: the occupancy math over synthetic transcripts with known
  876. // answers. It lives here rather than in a test file on purpose — a new
  877. // scripts/agent-eval/*.mjs scores into the self-query eval fixture's corpus.
  878. function selftest() {
  879. const { writeFileSync, mkdtempSync } = require0('fs');
  880. const { join } = require0('path');
  881. const { tmpdir } = require0('os');
  882. const dir = mkdtempSync(join(tmpdir(), 'cg-occ-'));
  883. let n = 0, failures = 0;
  884. const check = (name, got, want, tol) => {
  885. n++;
  886. const ok = Math.abs(got - want) <= tol;
  887. if (!ok) failures++;
  888. console.log(`${ok ? ' ok ' : ' FAIL'} ${name}: got ${Math.round(got)}, want ${want} ±${tol}`);
  889. };
  890. // Builders for the event shapes Claude Code actually emits.
  891. const req = (ctx, id, blocks) => blocks.map((b) => JSON.stringify({
  892. type: 'assistant',
  893. message: { id, content: [b], usage: { input_tokens: ctx, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, output_tokens: 2 } },
  894. }));
  895. const use = (id, name, input = {}) => ({ type: 'tool_use', id, name, input });
  896. const res = (id, chars) => JSON.stringify({
  897. type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: id, content: [{ type: 'text', text: 'x'.repeat(chars) }] }] },
  898. });
  899. const done = () => JSON.stringify({ type: 'result', subtype: 'success', duration_ms: 1000, total_cost_usd: 0.1, usage: {} });
  900. const write = (name, lines) => { const f = join(dir, name); writeFileSync(f, lines.join('\n') + '\n'); return f; };
  901. // 1. Attribution: ratio 2.5 chars/tok, two families, no shedding.
  902. // 10,000 explore chars over a 4,000-tok gap; 5,000 Read chars over 2,000.
  903. let f = write('basic.jsonl', [
  904. ...req(10000, 'm1', [use('t1', 'mcp__codegraph__codegraph_explore')]),
  905. res('t1', 10000),
  906. ...req(14000, 'm2', [use('t2', 'Read')]),
  907. res('t2', 5000),
  908. ...req(16000, 'm3', [{ type: 'text', text: 'done' }]),
  909. done(),
  910. ]);
  911. let o = parseSession([f]).occupancy;
  912. check('chars/token', o.charsPerToken * 1000, 2500, 30);
  913. check('codegraph residual', o.residual.codegraph, 4000, 60);
  914. check('Read residual', o.residual.read, 2000, 40);
  915. check('file-access residual', o.residualFileAccess, 2000, 40);
  916. check('final context', o.ctxFinal, 16000, 0);
  917. check('fixed base', o.ctxBase, 10000, 0);
  918. check('nothing evicted', o.evicted, 0, 1);
  919. // 2. Dedupe: thinking + tool_use are two events sharing one id and one usage.
  920. // Counting usage per event would report 5 requests instead of 3.
  921. f = write('dupe.jsonl', [
  922. ...req(10000, 'm1', [{ type: 'thinking', thinking: '' }, use('t1', 'mcp__codegraph__codegraph_explore')]),
  923. res('t1', 10000),
  924. ...req(14000, 'm2', [{ type: 'thinking', thinking: '' }, use('t2', 'Read')]),
  925. res('t2', 5000),
  926. ...req(16000, 'm3', [{ type: 'text', text: 'done' }]),
  927. done(),
  928. ]);
  929. let s = parseSession([f]);
  930. check('turns deduped by message.id', s.turns, 3, 0);
  931. check('codegraph residual (deduped)', s.occupancy.residual.codegraph, 4000, 60);
  932. // 3. Compaction: the boundary clears everything resident before it.
  933. f = write('compact.jsonl', [
  934. ...req(10000, 'm1', [use('t1', 'mcp__codegraph__codegraph_explore')]),
  935. res('t1', 10000),
  936. ...req(14000, 'm2', [use('t2', 'mcp__codegraph__codegraph_explore')]),
  937. JSON.stringify({ type: 'system', subtype: 'compact_boundary' }),
  938. res('t2', 5000),
  939. ...req(8000, 'm3', [{ type: 'text', text: 'done' }]),
  940. done(),
  941. ]);
  942. o = parseSession([f]).occupancy;
  943. check('post-compaction residual = last result only', o.residual.codegraph, 2000, 40);
  944. check('contributed still counts both', o.contributed.codegraph, 6000, 80);
  945. // 4. Micro-compaction: context grows less than the results added, so the
  946. // oldest result is shed first (FIFO) — here explore, leaving Read.
  947. f = write('micro.jsonl', [
  948. ...req(10000, 'm1', [use('t1', 'mcp__codegraph__codegraph_explore')]),
  949. res('t1', 10000),
  950. ...req(14000, 'm2', [use('t2', 'Read')]),
  951. res('t2', 10000),
  952. ...req(14500, 'm3', [{ type: 'text', text: 'done' }]), // +500 for 4,000 tok of Read
  953. done(),
  954. ]);
  955. o = parseSession([f]).occupancy;
  956. check('FIFO evicted the older codegraph result', o.residual.codegraph, 500, 60);
  957. check('newer Read result survives', o.residual.read, 4000, 60);
  958. check('eviction recorded', o.evicted, 3500, 60);
  959. // 5. Multi-turn stitching: a resumed segment continues the same context, and
  960. // a turn that calls no tool leaves the earlier residual in place.
  961. const a = write('seg1.jsonl', [
  962. ...req(10000, 'm1', [use('t1', 'mcp__codegraph__codegraph_explore')]),
  963. res('t1', 10000),
  964. ...req(14000, 'm2', [{ type: 'text', text: 'answer one' }]),
  965. done(),
  966. ]);
  967. const b = write('seg2.jsonl', [
  968. ...req(14600, 'm3', [{ type: 'text', text: 'answer two, from what is already here' }]),
  969. done(),
  970. ]);
  971. s = parseSession([a, b]);
  972. check('stitched turns', s.turns, 3, 0);
  973. check('residual carries into turn 2', s.occupancy.residual.codegraph, 4000, 60);
  974. check('stitched final context', s.occupancy.ctxFinal, 14600, 0);
  975. check('stitched cost sums segments', s.cost * 100, 20, 0.1);
  976. // ---- 6. Explore sufficiency: the bucket each explore call earns. --------
  977. const checkIs = (name, got, want) => {
  978. n++;
  979. const ok = got === want;
  980. if (!ok) failures++;
  981. console.log(`${ok ? ' ok ' : ' FAIL'} ${name}: got ${JSON.stringify(got)}, want ${JSON.stringify(want)}`);
  982. };
  983. const EXPLORE = 'mcp__codegraph__codegraph_explore';
  984. // An explore response's shape that matters here: one `**`path`**` section per
  985. // file whose source it returned, plus whatever else it named.
  986. const exploreRes = (id, paths, extra = '', isError = false) => JSON.stringify({
  987. type: 'user',
  988. message: {
  989. content: [{
  990. type: 'tool_result', tool_use_id: id, ...(isError ? { is_error: true } : {}),
  991. content: [{ type: 'text', text: paths.map((p) => `**\`${p}\`** — fn(function)\n\n1\tcode here\n`).join('\n') + extra }],
  992. }],
  993. },
  994. });
  995. const suff = (lines) => classifySufficiency(lines.map((l) => JSON.parse(l)));
  996. // explore → explore is insufficient; the second explore → a Read of a file it
  997. // RETURNED is the allocation bucket (this is the CG-22 express baseline).
  998. let sf = suff([
  999. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'res.send Content-Type ETag generation' })]),
  1000. exploreRes('e1', ['lib/response.js', 'lib/utils.js']),
  1001. ...req(12000, 'm2', [use('e2', EXPLORE, { query: 'response.js res.send function body' })]),
  1002. exploreRes('e2', ['lib/response.js']),
  1003. ...req(14000, 'm3', [use('r1', 'Read', { file_path: '/private/tmp/t-base/lib/response.js' })]),
  1004. res('r1', 3722),
  1005. ...req(15000, 'm4', [{ type: 'text', text: 'done' }]),
  1006. done(),
  1007. ]);
  1008. check('two answered explore calls', sf.answered, 2, 0);
  1009. checkIs('explore → explore = insufficient', sf.calls[0].bucket, 'explore_again');
  1010. checkIs('explore → Read of a returned file (abs path)', sf.calls[1].bucket, 'read_returned');
  1011. // A file the response NAMED but did not return is still a recall miss — and
  1012. // is flagged as named, since pointing without delivering is its own failure.
  1013. sf = suff([
  1014. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'q' })]),
  1015. exploreRes('e1', ['lib/response.js'], '\n**Flow**\n1. lib/router/index.js:42 handle\n'),
  1016. ...req(12000, 'm2', [use('r1', 'Read', { file_path: '/t/lib/router/index.js' })]),
  1017. res('r1', 100),
  1018. done(),
  1019. ]);
  1020. checkIs('explore → Read of a named-but-unreturned file', sf.calls[0].bucket, 'read_missed');
  1021. checkIs(' …flagged as named', sf.calls[0].named, true);
  1022. sf = suff([
  1023. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'q' })]),
  1024. exploreRes('e1', ['lib/response.js']),
  1025. ...req(12000, 'm2', [use('r1', 'Read', { file_path: '/t/lib/never/mentioned.js' })]),
  1026. res('r1', 100),
  1027. done(),
  1028. ]);
  1029. checkIs('explore → Read of a file never surfaced', sf.calls[0].bucket, 'read_missed');
  1030. checkIs(' …not flagged as named', sf.calls[0].named, false);
  1031. // Grep, and the shell equivalents of Read and Grep.
  1032. const oneShot = (next) => suff([
  1033. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'q' })]),
  1034. exploreRes('e1', ['lib/response.js']),
  1035. ...req(12000, 'm2', [next]),
  1036. res(next.id, 100),
  1037. done(),
  1038. ]).calls[0];
  1039. checkIs('explore → Grep', oneShot(use('g1', 'Grep', { pattern: 'send' })).bucket, 'search');
  1040. checkIs('explore → Glob', oneShot(use('g1', 'Glob', { pattern: '**/*.js' })).bucket, 'search');
  1041. checkIs('explore → Bash sed of a returned file',
  1042. oneShot(use('b1', 'Bash', { command: "sed -n '100,200p' lib/response.js" })).bucket, 'read_returned');
  1043. checkIs('explore → Bash grep', oneShot(use('b1', 'Bash', { command: 'grep -rn send lib/' })).bucket, 'search');
  1044. checkIs('explore → Bash that writes a file is not a read',
  1045. oneShot(use('b1', 'Bash', { command: "cat > /tmp/note.md <<'EOF'\nx\nEOF" })).bucket, 'sufficient');
  1046. checkIs('explore → Bash npm test = moved on',
  1047. oneShot(use('b1', 'Bash', { command: 'npm test' })).bucket, 'sufficient');
  1048. checkIs('explore → Edit = sufficient',
  1049. oneShot(use('x1', 'Edit', { file_path: '/t/lib/response.js' })).bucket, 'sufficient');
  1050. // No further tool call at all: the agent answered from the response.
  1051. sf = suff([
  1052. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'q' })]),
  1053. exploreRes('e1', ['lib/response.js']),
  1054. ...req(12000, 'm2', [{ type: 'text', text: 'here is how it works' }]),
  1055. done(),
  1056. ]);
  1057. checkIs('explore → final answer', sf.calls[0].bucket, 'sufficient');
  1058. checkIs(' …labelled as the final answer', sf.calls[0].next, '(final answer)');
  1059. // A Read issued in the SAME message as the explore predates its response, so
  1060. // it is not a verdict on it — step past it and count it separately.
  1061. sf = suff([
  1062. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'q' }), use('r1', 'Read', { file_path: '/t/lib/response.js' })]),
  1063. exploreRes('e1', ['lib/response.js']),
  1064. res('r1', 100),
  1065. ...req(12000, 'm2', [use('x1', 'Edit', { file_path: '/t/lib/response.js' })]),
  1066. res('x1', 20),
  1067. done(),
  1068. ]);
  1069. checkIs('same-message Read is not a reaction', sf.calls[0].bucket, 'sufficient');
  1070. check(' …counted as concurrent instead', sf.concurrent, 1, 0);
  1071. // ToolSearch/TodoWrite carry no signal — the Read behind them is the verdict.
  1072. sf = suff([
  1073. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'q' })]),
  1074. exploreRes('e1', ['lib/response.js']),
  1075. ...req(12000, 'm2', [use('t1', 'TodoWrite', {})]),
  1076. res('t1', 20),
  1077. ...req(13000, 'm3', [use('r1', 'Read', { file_path: '/t/lib/response.js' })]),
  1078. res('r1', 100),
  1079. done(),
  1080. ]);
  1081. checkIs('bookkeeping tools are stepped over', sf.calls[0].bucket, 'read_returned');
  1082. // A re-read of a file an EARLIER explore shipped is still an allocation miss:
  1083. // we returned it and clipped it wrong, just not on this call.
  1084. sf = suff([
  1085. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'first' })]),
  1086. exploreRes('e1', ['lib/response.js', 'lib/utils.js']),
  1087. ...req(11000, 'm2', [use('e2', EXPLORE, { query: 'second' })]),
  1088. exploreRes('e2', ['lib/response.js']),
  1089. ...req(12000, 'm3', [use('r1', 'Read', { file_path: '/t/lib/utils.js' })]),
  1090. res('r1', 400),
  1091. done(),
  1092. ]);
  1093. checkIs('re-read of an earlier explore’s file is allocation, not recall',
  1094. sf.calls[1].bucket, 'read_returned');
  1095. checkIs(' …and says which explore returned it',
  1096. sf.calls[1].next, 'Read utils.js (returned by an earlier explore)');
  1097. // A subagent's calls are interleaved into the same stream under
  1098. // `parent_tool_use_id` (verified on a real excalidraw run). They belong to
  1099. // their own thread: the parent's verdict is the delegation, judged by what
  1100. // the subagent actually did first — here, grepping for a file we never
  1101. // returned.
  1102. const sub = (parent, obj) => JSON.stringify({ ...JSON.parse(obj), parent_tool_use_id: parent });
  1103. sf = suff([
  1104. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'q' })]),
  1105. exploreRes('e1', ['lib/response.js']),
  1106. ...req(12000, 'm2', [use('a1', 'Agent', { subagent_type: 'Explore' })]),
  1107. ...req(0, 'sm1', [use('sb1', 'Bash', { command: 'grep -rn nonce lib/' })]).map((l) => sub('a1', l)),
  1108. sub('a1', res('sb1', 400)),
  1109. ...req(14000, 'm3', [{ type: 'text', text: 'done' }]),
  1110. res('a1', 900),
  1111. done(),
  1112. ]);
  1113. checkIs('delegation is judged by what the subagent did', sf.calls[0].bucket, 'search');
  1114. checkIs(' …and says so', sf.calls[0].next, 'Agent → Bash search');
  1115. // A subagent's Read must NOT be read as the parent's reaction to an explore
  1116. // the subagent never saw: the parent moved on, the subagent's own explore is
  1117. // judged inside its own thread.
  1118. sf = suff([
  1119. ...req(10000, 'm1', [use('a1', 'Agent', { subagent_type: 'Explore' })]),
  1120. ...req(0, 'sm1', [use('e1', EXPLORE, { query: 'sub q' })]).map((l) => sub('a1', l)),
  1121. sub('a1', exploreRes('e1', ['lib/response.js'])),
  1122. ...req(11000, 'm2', [use('e2', EXPLORE, { query: 'parent q' })]),
  1123. exploreRes('e2', ['lib/other.js']),
  1124. ...req(0, 'sm2', [use('sr1', 'Read', { file_path: '/t/lib/response.js' })]).map((l) => sub('a1', l)),
  1125. sub('a1', res('sr1', 400)),
  1126. ...req(13000, 'm3', [{ type: 'text', text: 'done' }]),
  1127. done(),
  1128. ]);
  1129. check('both threads bucketed', sf.answered, 2, 0);
  1130. checkIs('parent explore is not blamed for a subagent Read',
  1131. sf.calls.find((c) => c.query === 'parent q').bucket, 'sufficient');
  1132. checkIs('subagent explore is judged in its own thread',
  1133. sf.calls.find((c) => c.query === 'sub q').bucket, 'read_returned');
  1134. // An errored explore has no response to judge; it is counted, not bucketed.
  1135. sf = suff([
  1136. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'q' })]),
  1137. exploreRes('e1', [], 'not indexed', true),
  1138. ...req(12000, 'm2', [use('r1', 'Read', { file_path: '/t/lib/response.js' })]),
  1139. res('r1', 100),
  1140. done(),
  1141. ]);
  1142. check('errored explore is not bucketed', sf.answered, 0, 0);
  1143. check(' …but is counted', sf.errors, 1, 0);
  1144. // ---- 7. Allocation efficiency: which returned bytes the answer used. ----
  1145. // The response shape that matters here: a per-file section whose header lists
  1146. // `name(kind)`, and a body big enough that the shares are readable.
  1147. const alloc = (files, answer) => computeAllocation(
  1148. [files.map(([p, syms, size]) =>
  1149. `**\`${p}\`** — ${syms}\n\n\`\`\`js\n${'x'.repeat(size)}\n\`\`\`\n`).join('\n')
  1150. + '\n> Treat the code above as already read.\n'],
  1151. answer,
  1152. );
  1153. const round = (f) => Math.round(f * 1000) / 10;
  1154. // The #1500 shape: one file answers, two are noise. The answer names it by
  1155. // repo-relative path.
  1156. let al = alloc(
  1157. [['lib/response.js', 'send(function)', 4000], ['lib/express.js', 'app(variable)', 3000],
  1158. ['lib/view.js', 'View(class)', 3000]],
  1159. 'The Content-Type is decided in `lib/response.js:126` by `res.send`.',
  1160. );
  1161. check('allocation: one of three files cited', round(al.efficiency), 40.1, 0.6);
  1162. checkIs(' …cited file is flagged path', al.files.find((f) => /response/.test(f.path)).via, 'path');
  1163. checkIs(' …noise file is not', al.files.find((f) => /view/.test(f.path)).via, null);
  1164. check(' …files cited', al.filesUsed, 1, 0);
  1165. // A bare basename is how agents actually cite in prose — but only when the
  1166. // extension is one the envelope shipped, or `res.send` reads as a file too.
  1167. al = alloc([['lib/response.js', 'send(function)', 4000], ['lib/view.js', 'View(class)', 4000]],
  1168. 'It happens in response.js:134, inside `res.send`.');
  1169. check('bare basename counts as a path citation', round(al.efficiency), 50, 0.6);
  1170. al = alloc([['lib/response.js', 'send(function)', 4000], ['lib/view.js', 'View(class)', 4000]],
  1171. 'The `res.send` and `mime.contentType` calls do it.');
  1172. checkIs('a dotted expression is not a file citation', al.files.every((f) => f.via !== 'path'), true);
  1173. // Symbol citations: a code span naming a symbol the file DEFINES counts; a
  1174. // file that merely CALLS it does not (the excalidraw dragElements.ts case).
  1175. al = alloc([['src/mutateElement.ts', 'mutateElement(function)', 4000],
  1176. ['src/dragElements.ts', 'mutateElement(calls), updateCoords(function)', 4000]],
  1177. 'Mutation goes through `mutateElement`.');
  1178. checkIs('symbol citation credits the definer', al.files.find((f) => /mutateElement.ts/.test(f.path)).via, 'symbol');
  1179. checkIs(' …and not a caller of it', al.files.find((f) => /dragElements/.test(f.path)).via, null);
  1180. // An import binding (`variable`) loses to the real definition of the name.
  1181. al = alloc([['lib/utils.js', 'compileETag(function)', 4000],
  1182. ['lib/application.js', 'compileETag(variable), set(calls)', 4000]],
  1183. 'The generator comes from `compileETag`.');
  1184. checkIs('a definition beats an import alias of the same name',
  1185. al.files.find((f) => /utils/.test(f.path)).via, 'symbol');
  1186. checkIs(' …and the aliasing file is not credited',
  1187. al.files.find((f) => /application/.test(f.path)).via, null);
  1188. // A name carried by ≥3 returned files identifies none of them: crediting them
  1189. // all would bias the metric optimistic, the one direction it must not lean.
  1190. al = alloc([['a/one.js', 'handle(function)', 3000], ['a/two.js', 'handle(function)', 3000],
  1191. ['a/three.js', 'handle(function)', 3000]],
  1192. 'It all runs through `handle`.');
  1193. check('a symbol on 3 files attributes none of them', al.efficiency, 0, 0.001);
  1194. // Prose is not a citation — only a code span is.
  1195. al = alloc([['lib/response.js', 'sendResponse(function)', 4000], ['lib/view.js', 'View(class)', 4000]],
  1196. 'The sendResponse path handles it.');
  1197. check('an unquoted symbol in prose does not count', al.efficiency, 0, 0.001);
  1198. // Per-call: the run number pools, but call 2 being pure noise must still show.
  1199. const twoCalls = computeAllocation([
  1200. '**`lib/response.js`** — send(function)\n\n```js\n' + 'x'.repeat(4000) + '\n```\n\n> guidance\n',
  1201. '**`lib/view.js`** — View(class)\n\n```js\n' + 'x'.repeat(4000) + '\n```\n\n> guidance\n',
  1202. ], 'See `lib/response.js`.');
  1203. check('per-call: call 1 fully used', round(twoCalls.calls[0].efficiency), 100, 0.1);
  1204. check('per-call: call 2 fully wasted', round(twoCalls.calls[1].efficiency), 0, 0.1);
  1205. check('run pools both calls', round(twoCalls.efficiency), 50, 0.6);
  1206. // A file returned twice is charged twice — it occupied the window twice.
  1207. const twice = computeAllocation([
  1208. '**`lib/response.js`** — send(function)\n\n' + 'x'.repeat(4000) + '\n\n> g\n',
  1209. '**`lib/response.js`** — send(function)\n\n' + 'x'.repeat(4000) + '\n\n> g\n',
  1210. ], 'See `lib/response.js`.');
  1211. check('a re-returned file is charged both times', twice.envelope, 8078, 120);
  1212. check(' …and credited both times', round(twice.efficiency), 100, 0.1);
  1213. // The answer text itself: the `result` event, and the interactive fallback.
  1214. const ev = (l) => JSON.parse(l);
  1215. checkIs('final answer comes from the result event',
  1216. finalAnswerText([ev(done()), { type: 'result', result: 'the answer' }]), 'the answer');
  1217. checkIs('…else the last main-thread assistant text', finalAnswerText([
  1218. { type: 'assistant', message: { id: 'm1', content: [{ type: 'text', text: 'let me look' }] } },
  1219. { type: 'assistant', parent_tool_use_id: 'a1', message: { id: 's1', content: [{ type: 'text', text: 'subagent report' }] } },
  1220. { type: 'assistant', message: { id: 'm2', content: [{ type: 'text', text: 'the answer' }] } },
  1221. ]), 'the answer');
  1222. // End to end through parseSession, on the transcript shape a run really has.
  1223. f = write('alloc.jsonl', [
  1224. ...req(10000, 'm1', [use('e1', EXPLORE, { query: 'q' })]),
  1225. JSON.stringify({
  1226. type: 'user',
  1227. message: {
  1228. content: [{
  1229. type: 'tool_result', tool_use_id: 'e1',
  1230. content: [{
  1231. type: 'text',
  1232. text: '**`lib/response.js`** — send(function)\n\n' + 'x'.repeat(4000)
  1233. + '\n\n**`lib/view.js`** — View(class)\n\n' + 'x'.repeat(4000) + '\n\n> guidance\n',
  1234. }],
  1235. }],
  1236. },
  1237. }),
  1238. ...req(20000, 'm2', [{ type: 'text', text: 'done' }]),
  1239. JSON.stringify({
  1240. type: 'result', subtype: 'success', duration_ms: 1000, total_cost_usd: 0.1, usage: {},
  1241. result: 'It is decided in `lib/response.js` by `res.send`.',
  1242. }),
  1243. ]);
  1244. s = parseSession([f]);
  1245. check('parseSession reports allocation', round(s.allocation.efficiency), 50, 0.6);
  1246. checkIs(' …and the block renders', /efficiency\s+50\.\d%/.test(formatAllocation(s)), true);
  1247. checkIs('a run with no explore says so',
  1248. formatAllocation({ allocation: computeAllocation([], 'answer') }).includes('no codegraph_explore responses'), true);
  1249. console.log(`\n${n - failures}/${n} checks passed`);
  1250. return failures;
  1251. }
  1252. // `--selftest` needs sync fs helpers the module path doesn't import at top level.
  1253. function require0(m) { return process.getBuiltinModule(m); }
  1254. const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
  1255. if (isMain && process.argv.includes('--selftest')) process.exit(selftest() ? 1 : 0);
  1256. if (isMain) {
  1257. // `--answer <glob>` is repeatable and implies `--envelope`. Its VALUE is not a
  1258. // run file, so consume it here rather than letting the positional filter below
  1259. // mistake a glob for a log path.
  1260. const argv = process.argv.slice(2);
  1261. const files = [];
  1262. const answerGlobs = [];
  1263. let wantEnvelope = false;
  1264. for (let i = 0; i < argv.length; i++) {
  1265. if (argv[i] === '--envelope') wantEnvelope = true;
  1266. else if (argv[i] === '--answer') { answerGlobs.push(argv[++i]); wantEnvelope = true; }
  1267. else if (!argv[i].startsWith('--')) files.push(argv[i]);
  1268. }
  1269. if (!files.length) { console.error('usage: parse-run.mjs <run.jsonl> [run.t2.jsonl ...] [--envelope] [--answer <glob>]... | --selftest'); process.exit(1); }
  1270. const s = parseSession(files);
  1271. console.log(`\n=== ${files.map((f) => f.split('/').pop()).join(' + ')} ===`);
  1272. console.log(`codegraph tools exposed: ${s.initTools ? s.initTools.length : '?'}${s.raced ? ' [MCP COLD-START RACE — tool call hit "No such tool available"]' : ''}`);
  1273. if (s.cliContaminated) console.log(`!! ${s.cliContaminated} codegraph CLI call${s.cliContaminated === 1 ? '' : 's'} RETURNED OUTPUT via Bash — if this is a without-arm, the run is CONTAMINATED`);
  1274. else if (s.cliCalls) console.log(` (${s.cliCalls} codegraph CLI attempt${s.cliCalls === 1 ? '' : 's'} blocked — no output entered the window)`);
  1275. console.log(`\nTool calls (${s.toolCalls.length}):`);
  1276. console.log(' by type:', JSON.stringify(s.counts));
  1277. s.toolCalls.forEach((tc, i) => console.log(` ${i + 1}. ${tc}`));
  1278. if (s.result) {
  1279. const seg = s.results.length > 1 ? ` | ${s.results.length} segments (${s.results.map((r) => r.subtype).join(',')})` : '';
  1280. console.log(`\nResult: ${s.result.subtype} | duration ${s.dur.toFixed(0)}s | turns ${s.turns}${seg}`);
  1281. console.log(` tokens processed: ${s.processed.toLocaleString('en-US')} | cost $${s.cost.toFixed(3)}`);
  1282. }
  1283. console.log('');
  1284. console.log(formatOccupancy(s));
  1285. console.log('');
  1286. console.log(formatSufficiency(s));
  1287. console.log('');
  1288. console.log(formatAllocation(s));
  1289. if (wantEnvelope) {
  1290. console.log('');
  1291. console.log(formatEnvelope(s.exploreTexts, answerGlobs));
  1292. }
  1293. }