diff --git a/scripts/agent-eval/bench-readme.sh b/scripts/agent-eval/bench-readme.sh index 67c01b6..6dbb73c 100644 --- a/scripts/agent-eval/bench-readme.sh +++ b/scripts/agent-eval/bench-readme.sh @@ -14,6 +14,7 @@ set -uo pipefail H="$(cd "$(dirname "$0")" && pwd)" C="${CORPUS:-/tmp/codegraph-corpus}" RUNS="${RUNS:-4}" +RUN_FROM="${RUN_FROM:-1}" # extend an existing pass: RUN_FROM=3 RUNS=3 adds run3 only TURNS="${CG_TURNS:-3}" ROWS=( "vscode|How does the extension host communicate with the main process?|Where in that path would a message be dropped if the extension host crashes?|What would I need to change to add a new message type to that protocol?" @@ -24,7 +25,7 @@ ROWS=( "gin|How does gin route requests through its middleware chain?|Where is the 404 / no-route case handled in that same chain?|What would I change to add a per-route middleware that runs before the global ones?" "alamofire|How does Alamofire build, send, and validate a request?|Where does retry / interceptor logic hook into that path?|What would I change to add a new validation step to it?" ) -echo "### README A/B START $(date) RUNS=$RUNS TURNS=$TURNS" +echo "### README A/B START $(date) RUNS=$RUN_FROM..$RUNS TURNS=$TURNS" for row in "${ROWS[@]}"; do repo="${row%%|*}"; rest="${row#*|}" # Take the first $TURNS questions and join them with "||" for run-all.sh. @@ -36,7 +37,7 @@ for row in "${ROWS[@]}"; do q="$q$part"; n=$((n + 1)) done echo "===== $repo ($n turns) =====" - for run in $(seq 1 "$RUNS"); do + for run in $(seq "$RUN_FROM" "$RUNS"); do out="/tmp/ab-readme/$repo/run$run" mkdir -p "$out" AGENT_EVAL_OUT="$out" bash "$H/run-all.sh" "$C/$repo" "$q" headless > "$out/console.log" 2>&1