codegraph/scripts/agent-eval/bench-readme.sh

#!/usr/bin/env bash
# Re-run the README "Benchmark Results" A/B (with vs without codegraph) on the
# current build: the 7 README repos, same queries, RUNS per arm (default 4).
# Output → /tmp/ab-readme/<repo>/run<n>/run-headless-{with,without}[.tN].jsonl
# Aggregate with parse-bench-readme.mjs. Repos must be cloned + indexed under
# $CORPUS (default /tmp/codegraph-corpus) by the build under test.
#
# Each row is a THREE-TURN session: the README question, then two follow-ups
# that stay inside the same flow. Turns 2-3 are where residual context occupancy
# is actually charged — the first answer's tool output is still in the window,
# so the arms diverge on how much headroom each left behind. CG_TURNS=1 runs the
# README question alone (the original single-question A/B).
set -uo pipefail
H="$(cd "$(dirname "$0")" && pwd)"
C="${CORPUS:-/tmp/codegraph-corpus}"
RUNS="${RUNS:-4}"
RUN_FROM="${RUN_FROM:-1}"   # extend an existing pass: RUN_FROM=3 RUNS=3 adds run3 only
TURNS="${CG_TURNS:-3}"
ROWS=(
"vscode|How does the extension host communicate with the main process?|Where in that path would a message be dropped if the extension host crashes?|What would I need to change to add a new message type to that protocol?"
"excalidraw|How does Excalidraw render and update canvas elements?|Which part of that path decides whether a full re-render happens or an incremental one?|If I added a new element type, what in that render path would need to change?"
"django|How does Django's ORM build and execute a query from a QuerySet?|Where in that path is the SQL actually compiled into a string?|What would I change to add a new lookup type to that pipeline?"
"tokio|How does tokio schedule and run async tasks on its runtime?|Where does a task move between the local and the global queue in that path?|What in that path would I touch to add a per-task instrumentation hook?"
"okhttp|How does OkHttp process a request through its interceptor chain?|Where in that chain is the connection actually acquired?|What would I change to add a new interceptor stage before the cache?"
"gin|How does gin route requests through its middleware chain?|Where is the 404 / no-route case handled in that same chain?|What would I change to add a per-route middleware that runs before the global ones?"
"alamofire|How does Alamofire build, send, and validate a request?|Where does retry / interceptor logic hook into that path?|What would I change to add a new validation step to it?"
)
echo "### README A/B START $(date) RUNS=$RUN_FROM..$RUNS TURNS=$TURNS"
for row in "${ROWS[@]}"; do
  repo="${row%%|*}"; rest="${row#*|}"
  # Take the first $TURNS questions and join them with "||" for run-all.sh.
  q=""; n=0
  while [ "$n" -lt "$TURNS" ] && [ -n "$rest" ]; do
    part="${rest%%|*}"
    if [ "$rest" = "$part" ]; then rest=""; else rest="${rest#*|}"; fi
    [ -n "$q" ] && q="$q||"
    q="$q$part"; n=$((n + 1))
  done
  echo "===== $repo ($n turns) ====="
  for run in $(seq "$RUN_FROM" "$RUNS"); do
    out="/tmp/ab-readme/$repo/run$run"
    mkdir -p "$out"
    AGENT_EVAL_OUT="$out" bash "$H/run-all.sh" "$C/$repo" "$q" headless > "$out/console.log" 2>&1
    grep -E "^exit [0-9]" "$out/console.log" | sed 's/^/  /' || echo "  run$run: (no exit line)"
    grep -E "codegraph +[0-9,]+ tok|→ file-access" "$out/console.log" | sed 's/^/  /' || true
  done
done
echo "### README A/B DONE $(date)"