Sfoglia il codice sorgente

test(agent-eval): RUN_FROM, to extend a pass without redoing finished runs (CG-7)

Colby McHenry 1 mese fa
parent
commit
257a7b7327
1 ha cambiato i file con 3 aggiunte e 2 eliminazioni
  1. 3 2
      scripts/agent-eval/bench-readme.sh

+ 3 - 2
scripts/agent-eval/bench-readme.sh

@@ -14,6 +14,7 @@ set -uo pipefail
 H="$(cd "$(dirname "$0")" && pwd)"
 C="${CORPUS:-/tmp/codegraph-corpus}"
 RUNS="${RUNS:-4}"
+RUN_FROM="${RUN_FROM:-1}"   # extend an existing pass: RUN_FROM=3 RUNS=3 adds run3 only
 TURNS="${CG_TURNS:-3}"
 ROWS=(
 "vscode|How does the extension host communicate with the main process?|Where in that path would a message be dropped if the extension host crashes?|What would I need to change to add a new message type to that protocol?"
@@ -24,7 +25,7 @@ ROWS=(
 "gin|How does gin route requests through its middleware chain?|Where is the 404 / no-route case handled in that same chain?|What would I change to add a per-route middleware that runs before the global ones?"
 "alamofire|How does Alamofire build, send, and validate a request?|Where does retry / interceptor logic hook into that path?|What would I change to add a new validation step to it?"
 )
-echo "### README A/B START $(date) RUNS=$RUNS TURNS=$TURNS"
+echo "### README A/B START $(date) RUNS=$RUN_FROM..$RUNS TURNS=$TURNS"
 for row in "${ROWS[@]}"; do
   repo="${row%%|*}"; rest="${row#*|}"
   # Take the first $TURNS questions and join them with "||" for run-all.sh.
@@ -36,7 +37,7 @@ for row in "${ROWS[@]}"; do
     q="$q$part"; n=$((n + 1))
   done
   echo "===== $repo ($n turns) ====="
-  for run in $(seq 1 "$RUNS"); do
+  for run in $(seq "$RUN_FROM" "$RUNS"); do
     out="/tmp/ab-readme/$repo/run$run"
     mkdir -p "$out"
     AGENT_EVAL_OUT="$out" bash "$H/run-all.sh" "$C/$repo" "$q" headless > "$out/console.log" 2>&1