codegraph/scripts/agent-eval/probe-named-symbol.mjs

#!/usr/bin/env node
/**
 * "Did the symbol the agent NAMED actually render?" (CG-38).
 *
 * This is the measurement the whole CG-24 epic was missing. Every other probe
 * here scores the response in AGGREGATE — `probe-suite-envelope.mjs` measures how
 * much source came back, `probe-file-spend.mjs` measures whether the bytes went
 * to the files that earned them, `probe-allocation.mjs` measures group shares.
 * All three are green on a response that returns 25K of source from the right
 * file and still omits the one function the agent asked for by name. That is
 * exactly what CG-38 was: `queueMessage` at L1087 of a 1,414-line file, whose
 * file won rank #1 with 67% of the envelope, never rendered — the agent got a
 * same-stem `QueuedMessage` INTERFACE at L70 instead and had to Read the file.
 *
 * So the assertion here is per-SYMBOL and binary: for each named symbol, does its
 * definition line appear in the rendered source? Nothing else can substitute —
 * not the file being present, not its share, not its byte count.
 *
 * Usage (needs a current `npm run build`):
 *   node scripts/agent-eval/probe-named-symbol.mjs
 *   node scripts/agent-eval/probe-named-symbol.mjs --verbose
 *   # any indexed repo, ad hoc:
 *   node scripts/agent-eval/probe-named-symbol.mjs <repo> "<query>" sym1 sym2
 *
 * Exit code is 1 when any expected symbol is missing, so this can gate.
 */
import { cpSync, mkdtempSync, rmSync, readFileSync, existsSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve, dirname } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';

const HERE = dirname(fileURLToPath(import.meta.url));
const REPO = resolve(HERE, '..', '..');

const load = async (rel) => import(pathToFileURL(resolve(REPO, rel)).href);
const idxMod = await load('dist/index.js');
const toolsMod = await load('dist/mcp/tools.js');
const CodeGraph = idxMod.default?.default ?? idxMod.default ?? idxMod.CodeGraph;
const { ToolHandler } = toolsMod;

/**
 * The fixture cases. `symbols` are what the agent names; each must come back with
 * its DEFINITION rendered. The queries deliberately cover both shapes the bug was
 * reported on — a bare symbol bag and a prose question — because the failure had
 * a different cause on each and a fix for one does not imply the other.
 */
const FIXTURE = '__tests__/fixtures/tail-render-ts';
const CASES = [
  {
    id: 'tail-symbol-bag',
    why: 'two sibling closures past L1000, named directly; neither calls the other',
    query: 'queueMessage flushQueuedMessages',
    symbols: ['queueMessage', 'flushQueuedMessages'],
  },
  {
    id: 'tail-prose',
    why: 'same two symbols named inside a prose question',
    query: 'how does queueMessage hand its entries to flushQueuedMessages',
    symbols: ['queueMessage', 'flushQueuedMessages'],
  },
  {
    id: 'tail-with-decoy',
    why: 'the same-stem QueuedMessage interface at L70 must not stand in for the functions',
    query: 'explain queueMessage, removeQueuedMessage and flushQueuedMessages',
    symbols: ['queueMessage', 'removeQueuedMessage', 'flushQueuedMessages'],
  },
];

/** Every `<n>\t<text>` line number present in the response's source blocks. */
function renderedLines(response) {
  const out = new Set();
  for (const m of response.matchAll(/^(\d+)\t/gm)) out.add(Number(m[1]));
  return out;
}

/**
 * A symbol counts as rendered only when its DECLARATION line is among the lines
 * the response actually sent — not when its name merely appears somewhere (it
 * shows up in the section header symbol list and in call sites regardless, which
 * is precisely how this defect hid for a whole epic).
 */
function check(cg, response, names) {
  const lines = renderedLines(response);
  return names.map((name) => {
    const node = (cg.getNodesByName?.(name) ?? []).find((n) => n.startLine > 0);
    return {
      name,
      file: node?.filePath ?? '(not indexed)',
      line: node?.startLine ?? 0,
      rendered: !!node && lines.has(node.startLine),
    };
  });
}

async function runCase(root, { query, symbols }) {
  const cg = CodeGraph.openSync(root);
  try {
    const res = await new ToolHandler(cg).execute('codegraph_explore', { query });
    const response = res.content?.[0]?.text ?? '';
    return { response, results: check(cg, response, symbols) };
  } finally {
    try { cg.close?.(); } catch { /* already closed */ }
  }
}

const argv = process.argv.slice(2);
const verbose = argv.includes('--verbose');
const positional = argv.filter((a) => !a.startsWith('--'));

let failures = 0;
let checked = 0;

if (positional.length >= 3) {
  // Ad-hoc mode: <repo> "<query>" sym...
  const [repo, query, ...symbols] = positional;
  const { response, results } = await runCase(resolve(repo), { query, symbols });
  console.log(`\n${repo}\n  query "${query}"  ·  ${response.length} chars\n`);
  for (const r of results) {
    checked += 1;
    if (!r.rendered) failures += 1;
    console.log(`   ${r.rendered ? 'PASS' : 'FAIL'}  ${r.name}  ${r.file}:${r.line}`);
  }
} else {
  const src = join(REPO, FIXTURE);
  if (!existsSync(src)) {
    console.error(`fixture missing: ${src}`);
    process.exit(2);
  }
  const dir = mkdtempSync(join(tmpdir(), 'cg-named-'));
  try {
    cpSync(src, dir, { recursive: true });
    rmSync(join(dir, '.codegraph'), { recursive: true, force: true });
    const cg = CodeGraph.initSync(dir);
    await cg.indexAll();
    cg.close?.();

    console.log(`\ntail-render-ts  ·  agent-named symbols must render\n`);
    for (const c of CASES) {
      const { response, results } = await runCase(dir, c);
      console.log(`── ${c.id} — ${c.why}`);
      console.log(`   query   "${c.query}"`);
      console.log(`   response ${response.length.toLocaleString()} chars`);
      for (const r of results) {
        checked += 1;
        if (!r.rendered) failures += 1;
        console.log(`   ${r.rendered ? 'PASS' : 'FAIL'}  ${r.name} defined at ${r.file}:${r.line}`
          + (r.rendered ? '' : '  — DEFINITION NOT IN RESPONSE'));
      }
      if (verbose && failures) {
        const spans = [...renderedLines(response)].sort((a, b) => a - b);
        console.log(`   rendered lines: ${spans[0]}..${spans[spans.length - 1]} (${spans.length} lines)`);
      }
      console.log();
    }
  } finally {
    rmSync(dir, { recursive: true, force: true });
  }
}

console.log(failures === 0
  ? `Every agent-named symbol rendered (${checked} checked).`
  : `${failures} of ${checked} agent-named symbols did NOT render.`);
process.exit(failures === 0 ? 0 : 1);