0682681175
All agent A/B arms now run claude --model sonnet --effort high by default (MODEL/EFFORT env overrides exist). Sonnet is the deliberate floor model: codegraph's users attach whatever host they already run (Cursor Composer, Gemini, ...), and a stronger model's tool-use masks the salience problems a weaker one exposes — what lands on Sonnet generalizes up; Opus/Fable-only wins don't generalize down. Policy recorded in CLAUDE.md's validation methodology; 11 hardcoded --model opus call sites across 9 eval scripts switched to the env-overridable default. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
57 lines
2.9 KiB
Bash
Executable File
57 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Tool-surface ablation — run ONE repo+question under ONE arm.
|
|
#
|
|
# Arms vary (exposed codegraph tools, trace-first steering). Tools are trimmed
|
|
# SERVER-SIDE via CODEGRAPH_MCP_TOOLS in the MCP config's `env` block, so an
|
|
# ablated tool is genuinely absent from ListTools — no deferred-ToolSearch or
|
|
# denied-call confound (which --disallowedTools would introduce). Steering is
|
|
# injected with --append-system-prompt, so no rebuild of the shipped
|
|
# server-instructions is needed to A/B it.
|
|
#
|
|
# A control all tools no steering
|
|
# B steer all tools trace-first
|
|
# C no-explore hide explore trace-first
|
|
# D trace-centric hide explore+context trace-first
|
|
# E control-probe hide explore+context trace-first (caller passes a NON-flow Q)
|
|
#
|
|
# Usage: run-arms.sh <repo-path> "<question>" <A|B|C|D|E> [run-id]
|
|
set -uo pipefail
|
|
REPO="${1:?repo path}"; Q="${2:?question}"; ARM="${3:?arm A-E}"; RID="${4:-1}"
|
|
CG_BIN="${CG_BIN:-$(command -v codegraph)}"
|
|
OUT="${ARMS_OUT:-/tmp/arms}/$(basename "$REPO")"
|
|
mkdir -p "$OUT"
|
|
[ -n "$CG_BIN" ] || { echo "no codegraph binary (set CG_BIN)"; exit 1; }
|
|
[ -d "$REPO/.codegraph" ] || { echo "no .codegraph index at $REPO"; exit 1; }
|
|
|
|
STEER='Flow questions ("how does X reach/become Y", "trace the flow", request to handler, state to render): call codegraph_trace(from,to) FIRST — one call returns the whole path. Use codegraph_context/search only to locate the two endpoint symbols if you do not know them. Do NOT reconstruct the path with repeated search/callers/explore.'
|
|
KEEP_NO_EXPLORE="trace,search,node,context,callers,callees,impact,files,status"
|
|
KEEP_TRACE_CENTRIC="trace,search,node,callers,callees,impact,files,status"
|
|
|
|
case "$ARM" in
|
|
A|G|H|I) TOOLS=""; STEERING="" ;; # no steering; H = body-trace, I = body-trace + destination callees (sufficiency)
|
|
B|F) TOOLS=""; STEERING="$STEER" ;; # F = B's surface, run on the body-inlining trace build
|
|
C) TOOLS="$KEEP_NO_EXPLORE"; STEERING="$STEER" ;;
|
|
D|E) TOOLS="$KEEP_TRACE_CENTRIC"; STEERING="$STEER" ;;
|
|
*) echo "bad arm '$ARM' (want A|B|C|D|E)"; exit 1 ;;
|
|
esac
|
|
|
|
CFG="$OUT/mcp-$ARM.json"
|
|
if [ -n "$TOOLS" ]; then
|
|
cat > "$CFG" <<JSON
|
|
{"mcpServers":{"codegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"],"env":{"CODEGRAPH_MCP_TOOLS":"$TOOLS"}}}}
|
|
JSON
|
|
else
|
|
cat > "$CFG" <<JSON
|
|
{"mcpServers":{"codegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
|
|
JSON
|
|
fi
|
|
|
|
LOG="$OUT/$ARM-r$RID.jsonl"; ERR="$OUT/$ARM-r$RID.err"
|
|
ARGS=( -p "$Q" --output-format stream-json --verbose
|
|
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4
|
|
--strict-mcp-config --mcp-config "$CFG" )
|
|
[ -n "$STEERING" ] && ARGS+=( --append-system-prompt "$STEERING" )
|
|
|
|
( cd "$REPO" && claude "${ARGS[@]}" > "$LOG" 2>"$ERR" )
|
|
echo "[$(basename "$REPO") $ARM r$RID] exit $? -> $LOG ($(wc -l < "$LOG" | tr -d ' ') lines)"
|