codegraph/scripts/agent-eval/ab-impl.sh

#!/usr/bin/env bash
# Sufficiency A/B for an IMPLEMENTATION task (the agent edits): when it uses
# codegraph (explore/node) to understand before editing, does it still Read? Like
# ab-sufficiency.sh but copies+indexes a FRESH target per run (the agent mutates
# it), so runs don't see each other's edits.
#
# WITH codegraph (pre-warmed) vs WITHOUT (empty MCP), N runs each. Reports
# explore/node vs Read/Grep + the files Read, and whether the build still passes.
#
# Usage: ab-impl.sh <indexed-repo> "<task>" [runs] [build-cmd]
# Env: AGENT_EVAL_OUT (default: /tmp/ab-impl)
set -uo pipefail
REPO="${1:?usage: ab-impl.sh <indexed-repo> \"<task>\" [runs] [build-cmd]}"
Q="${2:?task required}"
RUNS="${3:-2}"
BUILD_CMD="${4:-}"
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
BIN="$ENGINE/dist/bin/codegraph.js"
OUT="${AGENT_EVAL_OUT:-/tmp/ab-impl}"
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
[ -d "$REPO/.codegraph" ] || { echo "no .codegraph index at $REPO"; exit 1; }
cleanup(){ pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null; }
trap cleanup EXIT
mkdir -p "$OUT"
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built engine"
echo "###### repo=$REPO  runs/arm=$RUNS"
echo "###### task=$Q"; echo
echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"

prewarm(){
  pkill -9 -f "serve --mcp --path $1" 2>/dev/null
  CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
}

analyze(){
  node -e '
    const fs=require("fs");
    const L=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
    let ex=0,nf=0,ns=0,oc=0,gr=0,ed=0,exposed="?";const reads=[];
    for(const l of L){try{const o=JSON.parse(l);
      if(o.type==="system"&&o.subtype==="init")exposed=(o.tools||[]).filter(t=>/codegraph/.test(t)).length;
      for(const b of (o.message?.content||[])){if(b.type!=="tool_use")continue;
        if(b.name==="mcp__codegraph__codegraph_explore")ex++;
        else if(b.name==="mcp__codegraph__codegraph_node"){if(b.input&&b.input.symbol)ns++;else nf++;}
        else if(/mcp__codegraph__/.test(b.name))oc++;
        else if(b.name==="Read")reads.push((b.input?.file_path||"").split("/").pop());
        else if(b.name==="Grep")gr++;
        else if(b.name==="Edit"||b.name==="Write")ed++;
      }}catch{}}
    console.log(`    explore=${ex} node[sym]=${ns} node[file]=${nf} other_cg=${oc} | Read=${reads.length}${reads.length?" ("+reads.join(", ")+")":""} Grep=${gr} Edit=${ed}  [cg exposed=${exposed}]`);
  ' "$1"
}

run(){ # label, withCodegraph(0/1)
  local label="$1" wcg="$2"
  for i in $(seq 1 "$RUNS"); do
    local tgt="$OUT/t-$label-$i" cfg="$OUT/mcp-$label.json"
    rm -rf "$tgt"
    rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .codegraph "$REPO/" "$tgt/"
    node "$BIN" init "$tgt" >/dev/null 2>&1
    if [ "$wcg" = "1" ]; then
      printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$cfg"
      prewarm "$tgt"
    else cp "$OUT/mcp-empty.json" "$cfg"; fi
    ( cd "$tgt" && claude -p "$Q" --output-format stream-json --verbose \
        --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
        --strict-mcp-config --mcp-config "$cfg" </dev/null > "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" )
    echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl"
    if [ -n "$BUILD_CMD" ]; then ( cd "$tgt" && eval "$BUILD_CMD" >/dev/null 2>&1 && echo "      build: PASS" || echo "      build: FAIL" ); fi
    pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
  done
  echo
}

echo "== WITH codegraph =="; run with 1
echo "== WITHOUT (Read/Grep only) =="; run without 0
echo "###### DONE: $OUT"