|
|
@@ -1,12 +1,33 @@
|
|
|
#!/usr/bin/env node
|
|
|
// Parse a Claude Code stream-json run log: tool-call sequence + token usage.
|
|
|
+//
|
|
|
+// With --envelope it also reports how the codegraph_explore responses the agent
|
|
|
+// received were DIVIDED across files — the per-file share of the source envelope.
|
|
|
+// That view is parsed out of the rendered markdown rather than the CG-4
|
|
|
+// diagnostic sidecar, so it works on ANY build (the sidecar only exists post-CG-4)
|
|
|
+// and is therefore the only way to measure both arms of a new-vs-baseline A/B the
|
|
|
+// same way. `--answer <glob>` (repeatable) marks the files that actually answer
|
|
|
+// the question, and the summary reports their combined share.
|
|
|
+//
|
|
|
+// Usage: parse-run.mjs <run.jsonl> [--envelope] [--answer <glob>]...
|
|
|
import { readFileSync } from 'fs';
|
|
|
-const file = process.argv[2];
|
|
|
+
|
|
|
+const argv = process.argv.slice(2);
|
|
|
+const answerGlobs = [];
|
|
|
+let file = null;
|
|
|
+let wantEnvelope = false;
|
|
|
+for (let i = 0; i < argv.length; i++) {
|
|
|
+ if (argv[i] === '--envelope') wantEnvelope = true;
|
|
|
+ else if (argv[i] === '--answer') { answerGlobs.push(argv[++i]); wantEnvelope = true; }
|
|
|
+ else if (!argv[i].startsWith('--') && file === null) file = argv[i];
|
|
|
+}
|
|
|
const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
|
|
|
|
|
|
const toolCalls = [];
|
|
|
let result = null;
|
|
|
let initTools = null;
|
|
|
+const exploreQueries = new Map(); // tool_use id -> query
|
|
|
+const exploreTexts = []; // response text, in call order
|
|
|
|
|
|
for (const line of lines) {
|
|
|
let ev;
|
|
|
@@ -23,6 +44,16 @@ for (const line of lines) {
|
|
|
else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`;
|
|
|
else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`;
|
|
|
toolCalls.push(`${block.name}${detail}`);
|
|
|
+ if (/codegraph_explore/.test(block.name)) exploreQueries.set(block.id, block.input?.query ?? '');
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (ev.type === 'user' && ev.message?.content) {
|
|
|
+ for (const block of ev.message.content) {
|
|
|
+ if (block.type === 'tool_result' && exploreQueries.has(block.tool_use_id)) {
|
|
|
+ exploreTexts.push(typeof block.content === 'string'
|
|
|
+ ? block.content
|
|
|
+ : (block.content ?? []).filter(c => c.type === 'text').map(c => c.text).join('\n'));
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -43,3 +74,53 @@ if (result) {
|
|
|
console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`);
|
|
|
console.log(` tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`);
|
|
|
}
|
|
|
+
|
|
|
+// ---- envelope share (opt-in) ------------------------------------------------
|
|
|
+
|
|
|
+if (wantEnvelope) {
|
|
|
+ // `tools/cache/**` -> /^tools\/cache\/.*$/ . Same semantics as probe-allocation.
|
|
|
+ // The `**` sentinel is written as an escape, never a literal NUL byte — a raw
|
|
|
+ // one makes git treat this whole script as binary and costs every future diff.
|
|
|
+ const glob2re = (glob) => {
|
|
|
+ const S = '\u0000';
|
|
|
+ const body = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
|
|
+ .replace(/\*\*/g, S).replace(/\*/g, '[^/]*').replaceAll(S, '.*');
|
|
|
+ return new RegExp(`^${body}$`);
|
|
|
+ };
|
|
|
+ const answerRes = answerGlobs.map(glob2re);
|
|
|
+ const isAnswer = (p) => answerRes.some(re => re.test(p));
|
|
|
+
|
|
|
+ // Each rendered file section starts with **`path`** — its bytes run to the next
|
|
|
+ // such header (or to the trailing guidance quote). Share is over the sum of the
|
|
|
+ // sections, i.e. of the source envelope the allocator divides.
|
|
|
+ const pooled = new Map();
|
|
|
+ let envelope = 0;
|
|
|
+ for (const text of exploreTexts) {
|
|
|
+ const re = /^\*\*`([^`]+)`\*\*/gm;
|
|
|
+ const marks = [];
|
|
|
+ let m;
|
|
|
+ while ((m = re.exec(text)) !== null) marks.push({ path: m[1], at: m.index });
|
|
|
+ if (!marks.length) continue;
|
|
|
+ const tail = text.indexOf('\n> ', marks[marks.length - 1].at);
|
|
|
+ const end = tail === -1 ? text.length : tail;
|
|
|
+ marks.forEach((mark, i) => {
|
|
|
+ const chars = (i + 1 < marks.length ? marks[i + 1].at : end) - mark.at;
|
|
|
+ pooled.set(mark.path, (pooled.get(mark.path) ?? 0) + chars);
|
|
|
+ envelope += chars;
|
|
|
+ });
|
|
|
+ }
|
|
|
+ const ranked = [...pooled.entries()]
|
|
|
+ .map(([path, chars]) => ({ path, chars, share: envelope ? chars / envelope : 0, answer: isAnswer(path) }))
|
|
|
+ .sort((a, b) => b.chars - a.chars);
|
|
|
+ const answerChars = ranked.filter(r => r.answer).reduce((s, r) => s + r.chars, 0);
|
|
|
+ const pct = (f) => `${(f * 100).toFixed(1)}%`;
|
|
|
+
|
|
|
+ console.log(`\nExplore envelope: ${envelope.toLocaleString('en-US')} chars over ${exploreTexts.length} response(s)`);
|
|
|
+ if (answerGlobs.length) {
|
|
|
+ console.log(` answer-set share: ${pct(envelope ? answerChars / envelope : 0)} | top file answers: ${ranked[0]?.answer ?? false}`);
|
|
|
+ }
|
|
|
+ for (const f of ranked.slice(0, 12)) {
|
|
|
+ console.log(` ${f.answer ? '*' : ' '} ${pct(f.share).padStart(6)} ${String(f.chars).padStart(6)} ${f.path}`);
|
|
|
+ }
|
|
|
+ if (ranked.length > 12) console.log(` … ${ranked.length - 12} more files`);
|
|
|
+}
|
Colby McHenry