codegraph/scripts/agent-eval/offload-eval-judge.mjs

#!/usr/bin/env node
// Accuracy judge. For each run in results.jsonl:
//   - end-to-end: agent finalAnswer vs verified ground truth (all arms)
//   - fidelity:   offload synthesized answer vs ground truth (offload arm only)
// Judge = claude -p sonnet --effort high, no tools, run from a neutral cwd,
// JSON-only verdicts. Writes judged.jsonl (one line per run, verdicts merged).
//
// Usage: judge.mjs --results <f> --truth <f> --out <f> [--concurrency 4]
import { readFileSync, writeFileSync, existsSync } from 'fs';
import { execFile } from 'child_process';

const A = {};
for (let i = 2; i < process.argv.length; i += 2) A[process.argv[i].replace(/^--/, '')] = process.argv[i + 1];
const results = readFileSync(A.results, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
const truth = JSON.parse(readFileSync(A.truth, 'utf8'));
const OUT = A.out || '/tmp/cg-offload-eval/judged.jsonl';
const CONC = Number(A.concurrency || 4);

function askJudge(prompt) {
  return new Promise((resolve) => {
    execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high',
      '--max-budget-usd', '0.5', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
      // Run from a neutral dir with no repo files so the judge can't "cheat" by reading source.
      { cwd: process.env.AGENT_EVAL_OUT || '/tmp', maxBuffer: 1 << 24, timeout: 120000 },
      (err, stdout) => {
        const raw = (stdout || '').trim();
        const m = raw.match(/\{[\s\S]*\}/);
        if (!m) return resolve({ verdict: 'error', score: null, note: (err ? 'exec ' + err.message : 'no json').slice(0, 80) });
        try { resolve(JSON.parse(m[0])); } catch { resolve({ verdict: 'error', score: null, note: 'parse fail' }); }
      });
  });
}

const e2ePrompt = (gt, ans) => `You are scoring whether an AI coding agent correctly answered a code-flow question about a repository. Judge ONLY against the verified ground truth. Do NOT use any tools.

QUESTION: ${gt.question}

VERIFIED GROUND TRUTH (the actual call path + files):
${gt.truth}

AGENT'S ANSWER:
${ans || '(empty)'}

Score how correct the agent's answer is vs the ground truth. A "pass" means it identifies the core mechanism and the major hops with the right files/symbols and makes no materially wrong claim. "partial" = right area but misses major hops or has notable errors. "fail" = wrong layer, fabricated, or misses the mechanism.
Output ONLY minified JSON, no prose, no code fences:
{"verdict":"pass|partial|fail","score":<0-100>,"missedHops":["..."],"wrongClaims":["..."],"note":"<=20 words"}`;

const fidPrompt = (gt, ans) => `You are scoring the FIDELITY of a machine-synthesized code-exploration answer against verified ground truth. The synthesized answer claims to trace a flow and cite file:line locations. Do NOT use any tools.

QUESTION: ${gt.question}

VERIFIED GROUND TRUTH (the actual call path + files):
${gt.truth}

SYNTHESIZED ANSWER (to score):
${ans || '(empty)'}

Judge: (1) is the traced call path correct vs ground truth? (2) are the cited files/symbols correct (not fabricated)? (3) if it gave a "Coverage:" verdict, was that verdict honest about what it actually covered? A confident WRONG trace is the worst outcome — penalize it harder than an honest "partial/not found".
Output ONLY minified JSON, no prose, no code fences:
{"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"missedHops":["..."],"note":"<=20 words"}`;

// Build the job list
const jobs = [];
for (const r of results) {
  const gt = truth[r.repo];
  if (!gt) { r._nojudge = true; continue; }
  jobs.push({ r, kind: 'e2e', prompt: e2ePrompt(gt, r.finalAnswer) });
  if (r.arm === 'offload' && Array.isArray(r.offloadAnswers))
    r.offloadAnswers.forEach((ans, i) => { if (ans && ans.trim()) jobs.push({ r, kind: 'fid', idx: i, prompt: fidPrompt(gt, ans) }); });
}
console.error(`judging ${jobs.length} verdicts across ${results.length} runs (concurrency ${CONC})...`);

let done = 0;
async function worker(queue) {
  while (queue.length) {
    const job = queue.shift();
    const v = await askJudge(job.prompt);
    if (job.kind === 'e2e') job.r.e2e = v; else (job.r._fid ??= []).push(v);
    console.error(`  [${++done}/${jobs.length}] ${job.r.repo}/${job.r.arm}#${job.r.rep} ${job.kind}: ${v.verdict}${v.score != null ? ' ' + v.score : ''}`);
  }
}
const q = [...jobs];
await Promise.all(Array.from({ length: CONC }, () => worker(q)));

// Aggregate per-answer fidelity verdicts into one fidelity object per offload run.
const medOf = (a) => { a = [...a].sort((x, y) => x - y); return a.length ? (a.length % 2 ? a[(a.length - 1) / 2] : (a[a.length / 2 - 1] + a[a.length / 2]) / 2) : null; };
for (const r of results) {
  if (r._fid?.length) {
    const scores = r._fid.map(v => v.score).filter(x => x != null);
    r.fidelity = {
      n: r._fid.length, scores,
      max: scores.length ? Math.max(...scores) : null,
      min: scores.length ? Math.min(...scores) : null,
      median: medOf(scores),
      anyFabrication: r._fid.some(v => v.fabrication === true),
      allCoverageHonest: r._fid.every(v => v.coverageHonest !== false),
      verdicts: r._fid.map(v => v.verdict),
    };
  }
  delete r._fid;
}
writeFileSync(OUT, results.map(r => JSON.stringify(r)).join('\n') + '\n');
console.error(`wrote ${OUT}`);