From 0682681175b82178a3bcc789bd6a186fab38ba58 Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Thu, 11 Jun 2026 19:39:16 -0500 Subject: [PATCH] =?UTF-8?q?chore(agent-eval):=20standing=20A/B=20model=20p?= =?UTF-8?q?olicy=20=E2=80=94=20sonnet=20+=20high=20effort,=20never=20Opus/?= =?UTF-8?q?Fable=20(#816)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All agent A/B arms now run claude --model sonnet --effort high by default (MODEL/EFFORT env overrides exist). Sonnet is the deliberate floor model: codegraph's users attach whatever host they already run (Cursor Composer, Gemini, ...), and a stronger model's tool-use masks the salience problems a weaker one exposes — what lands on Sonnet generalizes up; Opus/Fable-only wins don't generalize down. Policy recorded in CLAUDE.md's validation methodology; 11 hardcoded --model opus call sites across 9 eval scripts switched to the env-overridable default. Co-authored-by: Claude Opus 4.8 --- CLAUDE.md | 1 + scripts/agent-eval/ab-adoption.sh | 2 +- scripts/agent-eval/ab-hook.sh | 2 +- scripts/agent-eval/ab-impl.sh | 2 +- scripts/agent-eval/ab-new-vs-baseline.sh | 2 +- scripts/agent-eval/ab-sufficiency.sh | 2 +- scripts/agent-eval/bench-why-repo.sh | 2 +- scripts/agent-eval/run-agent.sh | 2 +- scripts/agent-eval/run-all.sh | 8 +++++--- scripts/agent-eval/run-arms.sh | 2 +- 10 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index bad199f..1831003 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -137,6 +137,7 @@ For each **language × framework**, validate on **small, medium, and large** rea 1. **Pick the canonical flow** for the framework ("how does X reach Y": state→render, request→handler→view, query→SQL, action→reducer→store…). 2. **Deterministic probes** (`scripts/agent-eval/probe-{node,explore}.mjs` against the built `dist/`): `codegraph_explore` with the flow's symbol names connects from→to end-to-end with no break (its Flow section shows the path); **no node explosion** (`select count(*) from nodes` stable before/after re-index); synthesized-edge **precision** spot-check (`select … where provenance='heuristic'`). 3. **Agent A/B** (`scripts/agent-eval/run-all.sh ""`): with vs without codegraph, **≥2 runs/arm** (run-to-run variance is large — never conclude from n=1). Record **duration, total tool calls, Read, Grep**. Optional forced-Read-0 sufficiency proof via the block-read hook (`scripts/agent-eval/hook-settings.json`). + - **Model policy — every A/B arm runs Claude with `--model sonnet --effort high`. Always. Never Opus/Fable.** All `scripts/agent-eval/*.sh` default to this (`MODEL`/`EFFORT` env override exists — don't raise it without an explicit reason from the maintainer). Two reasons, and the second matters more than cost: (a) Sonnet doesn't burn tokens; (b) **Sonnet is the deliberate floor model** — codegraph's real users attach it to whatever agent they already run (Cursor Composer, Gemini, etc.), so we validate on a "dumber" model on purpose: a stronger model's tool-use covers up the salience/sufficiency problems a weaker one exposes. An affordance that lands on Sonnet generalizes up to every host; one that only works on Opus/Fable doesn't generalize down to the agents most users actually have. Both arms always use the same model. - **MCP attach is a startup-latency issue, not a hard block.** On a multi-step task the agent dives into Read/grep before codegraph finishes its ~2-3s startup (worse when the eval is itself run nested inside a Claude session, under CPU contention), so it runs with no codegraph. Fix: **pre-warm a persistent daemon** for the target (`CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS` high; spawn `serve --mcp --path "" [baseline-ref]` (it bakes in the pre-warm). 4. **Pass bar:** a normal flow question reaches **~0 Read/Grep within the repo's explore-call budget**, runs **faster** than without-codegraph, and shows **no regression on a control repo**. Record the numbers in `docs/design/dynamic-dispatch-coverage-playbook.md` (the coverage matrix). diff --git a/scripts/agent-eval/ab-adoption.sh b/scripts/agent-eval/ab-adoption.sh index eabf802..d5c6dc2 100644 --- a/scripts/agent-eval/ab-adoption.sh +++ b/scripts/agent-eval/ab-adoption.sh @@ -71,7 +71,7 @@ run_arm() { # label, N echo "----- [$label] run $i -----" ( cd "$tgt" && claude -p "$TASK" \ --output-format stream-json --verbose --permission-mode bypassPermissions \ - --model opus --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \ + --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \ "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" ) count "$OUT/run-$label-$i.jsonl" pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null diff --git a/scripts/agent-eval/ab-hook.sh b/scripts/agent-eval/ab-hook.sh index 8c1af32..0e46fcc 100644 --- a/scripts/agent-eval/ab-hook.sh +++ b/scripts/agent-eval/ab-hook.sh @@ -72,7 +72,7 @@ run_one() { # arm-label, run-index, use-hook(0|1) # array expansion otherwise, which would skip the no-hook arm's claude run. ( cd "$tgt" && claude -p "$TASK" \ --output-format stream-json --verbose --permission-mode bypassPermissions \ - --model opus --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" ${extra[@]+"${extra[@]}"} \ + --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" ${extra[@]+"${extra[@]}"} \ "$OUT/run-$label-$idx.jsonl" 2>"$OUT/run-$label-$idx.err" ) node "$PARSE" "$OUT/run-$label-$idx.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label-$idx.jsonl)" pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null diff --git a/scripts/agent-eval/ab-impl.sh b/scripts/agent-eval/ab-impl.sh index c5c23b5..b6f219b 100644 --- a/scripts/agent-eval/ab-impl.sh +++ b/scripts/agent-eval/ab-impl.sh @@ -64,7 +64,7 @@ run(){ # label, withCodegraph(0/1) prewarm "$tgt" else cp "$OUT/mcp-empty.json" "$cfg"; fi ( cd "$tgt" && claude -p "$Q" --output-format stream-json --verbose \ - --permission-mode bypassPermissions --model opus --max-budget-usd 4 \ + --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \ --strict-mcp-config --mcp-config "$cfg" "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" ) echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl" if [ -n "$BUILD_CMD" ]; then ( cd "$tgt" && eval "$BUILD_CMD" >/dev/null 2>&1 && echo " build: PASS" || echo " build: FAIL" ); fi diff --git a/scripts/agent-eval/ab-new-vs-baseline.sh b/scripts/agent-eval/ab-new-vs-baseline.sh index 7e5cc84..b7fe4e7 100755 --- a/scripts/agent-eval/ab-new-vs-baseline.sh +++ b/scripts/agent-eval/ab-new-vs-baseline.sh @@ -75,7 +75,7 @@ run_arm() { # label, target-copy echo "############## ARM [$label] ##############" ( cd "$tgt" && claude -p "$TASK" \ --output-format stream-json --verbose --permission-mode bypassPermissions \ - --model opus --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \ + --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \ "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" ) node "$PARSE" "$OUT/run-$label.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label.jsonl)" pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null diff --git a/scripts/agent-eval/ab-sufficiency.sh b/scripts/agent-eval/ab-sufficiency.sh index 0662536..3db4ff1 100644 --- a/scripts/agent-eval/ab-sufficiency.sh +++ b/scripts/agent-eval/ab-sufficiency.sh @@ -66,7 +66,7 @@ run(){ # label, cfg, prewarm(0/1) for i in $(seq 1 "$RUNS"); do [ "$pw" = "1" ] && prewarm ( cd "$TGT" && claude -p "$Q" --output-format stream-json --verbose \ - --permission-mode bypassPermissions --model opus --max-budget-usd 4 \ + --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \ --strict-mcp-config --mcp-config "$cfg" "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" ) echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl" done diff --git a/scripts/agent-eval/bench-why-repo.sh b/scripts/agent-eval/bench-why-repo.sh index 2bbedf8..2e26a2f 100644 --- a/scripts/agent-eval/bench-why-repo.sh +++ b/scripts/agent-eval/bench-why-repo.sh @@ -15,7 +15,7 @@ printf '{"mcpServers":{"codegraph":{"command":"%s","args":["serve","--mcp","--pa for i in $(seq 1 "$N"); do pkill -f "serve --mcp" 2>/dev/null; sleep 1; rm -f "$REPO/.codegraph/daemon.sock" ( cd "$REPO" && claude -p "$Q$WHY" --output-format stream-json --verbose \ - --permission-mode bypassPermissions --model opus --effort "${EFFORT:-high}" --max-budget-usd 4 \ + --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \ --strict-mcp-config --mcp-config "$OUT/cg.json" > "$OUT/with$i.jsonl" 2>"$OUT/with$i.err" ) echo "WITH run $i: exit $? ($(wc -l < "$OUT/with$i.jsonl" | tr -d ' ') lines)" done diff --git a/scripts/agent-eval/run-agent.sh b/scripts/agent-eval/run-agent.sh index b599c43..abbee87 100755 --- a/scripts/agent-eval/run-agent.sh +++ b/scripts/agent-eval/run-agent.sh @@ -25,7 +25,7 @@ cd "$REPO" || exit 1 claude -p "$PROMPT" \ --output-format stream-json --verbose \ --permission-mode bypassPermissions \ - --model opus \ + --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \ --max-budget-usd 2 \ --strict-mcp-config --mcp-config "$MCP_CONFIG" \ > "$OUT" 2>"$OUT_DIR/run-${LABEL}.err" diff --git a/scripts/agent-eval/run-all.sh b/scripts/agent-eval/run-all.sh index 4b40dce..b682925 100755 --- a/scripts/agent-eval/run-all.sh +++ b/scripts/agent-eval/run-all.sh @@ -7,6 +7,8 @@ # Usage: run-all.sh "" [headless|tmux|all] # Env: CG_BIN codegraph binary (default: command -v codegraph) # AGENT_EVAL_OUT output dir (default: /tmp/agent-eval) +# MODEL / EFFORT claude model/effort (default: sonnet / high — the +# standing A/B policy; see CLAUDE.md, don't raise) set -uo pipefail REPO="${1:?usage: run-all.sh \"\" [headless|tmux|all]}" @@ -39,7 +41,7 @@ headless() { ( cd "$REPO" && claude -p "$Q" \ --output-format stream-json --verbose \ --permission-mode bypassPermissions \ - --model opus \ + --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \ --max-budget-usd 4 \ --strict-mcp-config --mcp-config "$cfg" \ > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" ) @@ -56,11 +58,11 @@ fi if [ "$MODE" = tmux ] || [ "$MODE" = all ]; then echo "############################## INTERACTIVE [with] ##############################" - CLAUDE_EXTRA_ARGS="--model opus --strict-mcp-config --mcp-config $OUT/mcp-codegraph.json" \ + CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-codegraph.json" \ bash "$HARNESS/itrun.sh" "$REPO" "int-with" "$Q" 2>&1 || echo "[itrun WITH failed]" echo echo "############################## INTERACTIVE [without] ##############################" - CLAUDE_EXTRA_ARGS="--model opus --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \ + CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \ bash "$HARNESS/itrun.sh" "$REPO" "int-without" "$Q" 2>&1 || echo "[itrun WITHOUT failed]" echo fi diff --git a/scripts/agent-eval/run-arms.sh b/scripts/agent-eval/run-arms.sh index af3da6d..48d4cf8 100755 --- a/scripts/agent-eval/run-arms.sh +++ b/scripts/agent-eval/run-arms.sh @@ -48,7 +48,7 @@ fi LOG="$OUT/$ARM-r$RID.jsonl"; ERR="$OUT/$ARM-r$RID.err" ARGS=( -p "$Q" --output-format stream-json --verbose - --permission-mode bypassPermissions --model opus --max-budget-usd 4 + --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$CFG" ) [ -n "$STEERING" ] && ARGS+=( --append-system-prompt "$STEERING" )