github/codegraph
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(agent-eval): standing A/B model policy — sonnet + high effort, never Opus/Fable (#816)

Browse Source 
All agent A/B arms now run claude --model sonnet --effort high by default
(MODEL/EFFORT env overrides exist). Sonnet is the deliberate floor model:
codegraph's users attach whatever host they already run (Cursor Composer,
Gemini, ...), and a stronger model's tool-use masks the salience problems a
weaker one exposes — what lands on Sonnet generalizes up; Opus/Fable-only
wins don't generalize down. Policy recorded in CLAUDE.md's validation
methodology; 11 hardcoded --model opus call sites across 9 eval scripts
switched to the env-overridable default.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
2026-06-11 19:39:16 -05:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 823ffd1c3d
commit 0682681175
10 changed files with 14 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/agent-eval/ab-adoption.sh
+1 -1
View File
@@ -71,7 +71,7 @@ run_arm() { # label, N
echo "----- [$label] run $i -----"
( cd "$tgt" && claude -p "$TASK" \
--output-format stream-json --verbose --permission-mode bypassPermissions \
--model opus --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
</dev/null > "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" )
count "$OUT/run-$label-$i.jsonl"
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
scripts/agent-eval/ab-hook.sh
+1 -1
View File
@@ -72,7 +72,7 @@ run_one() { # arm-label, run-index, use-hook(0|1)
# array expansion otherwise, which would skip the no-hook arm's claude run.
( cd "$tgt" && claude -p "$TASK" \
--output-format stream-json --verbose --permission-mode bypassPermissions \
--model opus --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" ${extra[@]+"${extra[@]}"} \
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" ${extra[@]+"${extra[@]}"} \
</dev/null > "$OUT/run-$label-$idx.jsonl" 2>"$OUT/run-$label-$idx.err" )
node "$PARSE" "$OUT/run-$label-$idx.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label-$idx.jsonl)"
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
scripts/agent-eval/ab-impl.sh
+1 -1
View File
@@ -64,7 +64,7 @@ run(){ # label, withCodegraph(0/1)
prewarm "$tgt"
else cp "$OUT/mcp-empty.json" "$cfg"; fi
( cd "$tgt" && claude -p "$Q" --output-format stream-json --verbose \
--permission-mode bypassPermissions --model opus --max-budget-usd 4 \
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
--strict-mcp-config --mcp-config "$cfg" </dev/null > "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" )
echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl"
if [ -n "$BUILD_CMD" ]; then ( cd "$tgt" && eval "$BUILD_CMD" >/dev/null 2>&1 && echo " build: PASS" || echo " build: FAIL" ); fi
scripts/agent-eval/ab-new-vs-baseline.sh
+1 -1
View File
@@ -75,7 +75,7 @@ run_arm() { # label, target-copy
echo "############## ARM [$label] ##############"
( cd "$tgt" && claude -p "$TASK" \
--output-format stream-json --verbose --permission-mode bypassPermissions \
--model opus --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
</dev/null > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
node "$PARSE" "$OUT/run-$label.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label.jsonl)"
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
scripts/agent-eval/ab-sufficiency.sh
+1 -1
View File
@@ -66,7 +66,7 @@ run(){ # label, cfg, prewarm(0/1)
for i in $(seq 1 "$RUNS"); do
[ "$pw" = "1" ] && prewarm
( cd "$TGT" && claude -p "$Q" --output-format stream-json --verbose \
--permission-mode bypassPermissions --model opus --max-budget-usd 4 \
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
--strict-mcp-config --mcp-config "$cfg" </dev/null > "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" )
echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl"
done
scripts/agent-eval/bench-why-repo.sh
+1 -1
View File
@@ -15,7 +15,7 @@ printf '{"mcpServers":{"codegraph":{"command":"%s","args":["serve","--mcp","--pa
for i in $(seq 1 "$N"); do
pkill -f "serve --mcp" 2>/dev/null; sleep 1; rm -f "$REPO/.codegraph/daemon.sock"
( cd "$REPO" && claude -p "$Q$WHY" --output-format stream-json --verbose \
--permission-mode bypassPermissions --model opus --effort "${EFFORT:-high}" --max-budget-usd 4 \
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
--strict-mcp-config --mcp-config "$OUT/cg.json" > "$OUT/with$i.jsonl" 2>"$OUT/with$i.err" )
echo "WITH run $i: exit $? ($(wc -l < "$OUT/with$i.jsonl" | tr -d ' ') lines)"
done
scripts/agent-eval/run-agent.sh
+1 -1
View File
@@ -25,7 +25,7 @@ cd "$REPO" || exit 1
claude -p "$PROMPT" \
--output-format stream-json --verbose \
--permission-mode bypassPermissions \
--model opus \
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
--max-budget-usd 2 \
--strict-mcp-config --mcp-config "$MCP_CONFIG" \
> "$OUT" 2>"$OUT_DIR/run-${LABEL}.err"
scripts/agent-eval/run-all.sh
+5 -3
View File
@@ -7,6 +7,8 @@
# Usage: run-all.sh <repo-path> "<question>" [headless|tmux|all]
# Env: CG_BIN codegraph binary (default: command -v codegraph)
# AGENT_EVAL_OUT output dir (default: /tmp/agent-eval)
# MODEL / EFFORT claude model/effort (default: sonnet / high — the
# standing A/B policy; see CLAUDE.md, don't raise)
set -uo pipefail
REPO="${1:?usage: run-all.sh <repo-path> \"<question>\" [headless|tmux|all]}"
@@ -39,7 +41,7 @@ headless() {
( cd "$REPO" && claude -p "$Q" \
--output-format stream-json --verbose \
--permission-mode bypassPermissions \
--model opus \
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
--max-budget-usd 4 \
--strict-mcp-config --mcp-config "$cfg" \
> "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
@@ -56,11 +58,11 @@ fi
if [ "$MODE" = tmux ] || [ "$MODE" = all ]; then
echo "############################## INTERACTIVE [with] ##############################"
CLAUDE_EXTRA_ARGS="--model opus --strict-mcp-config --mcp-config $OUT/mcp-codegraph.json" \
CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-codegraph.json" \
bash "$HARNESS/itrun.sh" "$REPO" "int-with" "$Q" 2>&1 || echo "[itrun WITH failed]"
echo
echo "############################## INTERACTIVE [without] ##############################"
CLAUDE_EXTRA_ARGS="--model opus --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \
CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \
bash "$HARNESS/itrun.sh" "$REPO" "int-without" "$Q" 2>&1 || echo "[itrun WITHOUT failed]"
echo
fi
scripts/agent-eval/run-arms.sh
+1 -1
View File
@@ -48,7 +48,7 @@ fi
LOG="$OUT/$ARM-r$RID.jsonl"; ERR="$OUT/$ARM-r$RID.err"
ARGS=( -p "$Q" --output-format stream-json --verbose
--permission-mode bypassPermissions --model opus --max-budget-usd 4
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4
--strict-mcp-config --mcp-config "$CFG" )
[ -n "$STEERING" ] && ARGS+=( --append-system-prompt "$STEERING" )