diff --git a/scripts/agent-eval/ab-new-vs-baseline.sh b/scripts/agent-eval/ab-new-vs-baseline.sh index 31e5080..53b57a4 100755 --- a/scripts/agent-eval/ab-new-vs-baseline.sh +++ b/scripts/agent-eval/ab-new-vs-baseline.sh @@ -21,7 +21,19 @@ # a repo with a .codegraph index (copied per arm) # "" an implementation task, e.g. "Add X to Y and wire it through" # [baseline-ref] git ref for the BEFORE build (default: HEAD~1) -# Env: AGENT_EVAL_OUT (default: /tmp/ab-new-vs-baseline) +# Env: +# AGENT_EVAL_OUT output dir (default: /tmp/ab-new-vs-baseline) +# RUNS runs per arm (default 1). Run-to-run variance is large — +# use >=2 and report the range, never a single run. Both arms +# build/index ONCE and then run RUNS times, so raising this is +# far cheaper than re-invoking the script. +# MODEL / EFFORT default sonnet / high. Never raise without a reason: sonnet +# is the deliberate floor model (see CLAUDE.md). +# +# Both arms run with CODEGRAPH_NO_PROMPT_HOOK=1: the machine's ambient +# UserPromptSubmit front-load hook resolves to whichever build is currently in +# dist/, so leaving it on injects context through a second, uncontrolled channel +# and confounds the tool-call counts this script exists to compare. set -uo pipefail TARGET="${1:?usage: ab-new-vs-baseline.sh \"\" [baseline-ref]}" @@ -67,18 +79,27 @@ prewarm() { # target — spawn a persistent daemon (current $BIN) and wait for i && echo " daemon warm: $1" || echo " WARN: daemon never bound for $1 (arm may run without codegraph)" } -run_arm() { # label, target-copy +run_arm() { # label, target-copy — runs the task $RUNS times against one build local label="$1" tgt="$2" c="$OUT/mcp-$1.json" # Connect to the pre-warmed daemon; skip the startup re-exec for a fast attach. - printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c" - prewarm "$tgt" + # CODEGRAPH_EXPLORE_DEBUG points explore's per-file allocation diagnostic at a + # sidecar (no-op on builds predating it; never perturbs the response). + printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_EXPLORE_DEBUG=%s","node","%s","serve","--mcp","--path","%s"]}}}' \ + "$OUT/explore-$label.jsonl" "$BIN" "$tgt" > "$c" + rm -f "$OUT/explore-$label.jsonl" echo "############## ARM [$label] ##############" - ( cd "$tgt" && claude -p "$TASK" \ - --output-format stream-json --verbose --permission-mode bypassPermissions \ - --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \ - "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" ) - node "$PARSE" "$OUT/run-$label.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label.jsonl)" - pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null + for i in $(seq 1 "${RUNS:-1}"); do + # Re-warm per run: the previous run's daemon is killed below, and a cold + # attach is exactly the failure this pre-warm exists to prevent. + prewarm "$tgt" + ( cd "$tgt" && CODEGRAPH_NO_PROMPT_HOOK=1 claude -p "$TASK" \ + --output-format stream-json --verbose --permission-mode bypassPermissions \ + --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \ + "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" ) + echo "-- run $i --" + node "$PARSE" "$OUT/run-$label-$i.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label-$i.jsonl)" + pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null + done echo } diff --git a/scripts/agent-eval/parse-run.mjs b/scripts/agent-eval/parse-run.mjs index 6d64d58..a78065a 100644 --- a/scripts/agent-eval/parse-run.mjs +++ b/scripts/agent-eval/parse-run.mjs @@ -1,12 +1,33 @@ #!/usr/bin/env node // Parse a Claude Code stream-json run log: tool-call sequence + token usage. +// +// With --envelope it also reports how the codegraph_explore responses the agent +// received were DIVIDED across files — the per-file share of the source envelope. +// That view is parsed out of the rendered markdown rather than the CG-4 +// diagnostic sidecar, so it works on ANY build (the sidecar only exists post-CG-4) +// and is therefore the only way to measure both arms of a new-vs-baseline A/B the +// same way. `--answer ` (repeatable) marks the files that actually answer +// the question, and the summary reports their combined share. +// +// Usage: parse-run.mjs [--envelope] [--answer ]... import { readFileSync } from 'fs'; -const file = process.argv[2]; + +const argv = process.argv.slice(2); +const answerGlobs = []; +let file = null; +let wantEnvelope = false; +for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--envelope') wantEnvelope = true; + else if (argv[i] === '--answer') { answerGlobs.push(argv[++i]); wantEnvelope = true; } + else if (!argv[i].startsWith('--') && file === null) file = argv[i]; +} const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean); const toolCalls = []; let result = null; let initTools = null; +const exploreQueries = new Map(); // tool_use id -> query +const exploreTexts = []; // response text, in call order for (const line of lines) { let ev; @@ -23,6 +44,16 @@ for (const line of lines) { else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`; else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`; toolCalls.push(`${block.name}${detail}`); + if (/codegraph_explore/.test(block.name)) exploreQueries.set(block.id, block.input?.query ?? ''); + } + } + } + if (ev.type === 'user' && ev.message?.content) { + for (const block of ev.message.content) { + if (block.type === 'tool_result' && exploreQueries.has(block.tool_use_id)) { + exploreTexts.push(typeof block.content === 'string' + ? block.content + : (block.content ?? []).filter(c => c.type === 'text').map(c => c.text).join('\n')); } } } @@ -43,3 +74,53 @@ if (result) { console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`); console.log(` tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`); } + +// ---- envelope share (opt-in) ------------------------------------------------ + +if (wantEnvelope) { + // `tools/cache/**` -> /^tools\/cache\/.*$/ . Same semantics as probe-allocation. + // The `**` sentinel is written as an escape, never a literal NUL byte — a raw + // one makes git treat this whole script as binary and costs every future diff. + const glob2re = (glob) => { + const S = '\u0000'; + const body = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&') + .replace(/\*\*/g, S).replace(/\*/g, '[^/]*').replaceAll(S, '.*'); + return new RegExp(`^${body}$`); + }; + const answerRes = answerGlobs.map(glob2re); + const isAnswer = (p) => answerRes.some(re => re.test(p)); + + // Each rendered file section starts with **`path`** — its bytes run to the next + // such header (or to the trailing guidance quote). Share is over the sum of the + // sections, i.e. of the source envelope the allocator divides. + const pooled = new Map(); + let envelope = 0; + for (const text of exploreTexts) { + const re = /^\*\*`([^`]+)`\*\*/gm; + const marks = []; + let m; + while ((m = re.exec(text)) !== null) marks.push({ path: m[1], at: m.index }); + if (!marks.length) continue; + const tail = text.indexOf('\n> ', marks[marks.length - 1].at); + const end = tail === -1 ? text.length : tail; + marks.forEach((mark, i) => { + const chars = (i + 1 < marks.length ? marks[i + 1].at : end) - mark.at; + pooled.set(mark.path, (pooled.get(mark.path) ?? 0) + chars); + envelope += chars; + }); + } + const ranked = [...pooled.entries()] + .map(([path, chars]) => ({ path, chars, share: envelope ? chars / envelope : 0, answer: isAnswer(path) })) + .sort((a, b) => b.chars - a.chars); + const answerChars = ranked.filter(r => r.answer).reduce((s, r) => s + r.chars, 0); + const pct = (f) => `${(f * 100).toFixed(1)}%`; + + console.log(`\nExplore envelope: ${envelope.toLocaleString('en-US')} chars over ${exploreTexts.length} response(s)`); + if (answerGlobs.length) { + console.log(` answer-set share: ${pct(envelope ? answerChars / envelope : 0)} | top file answers: ${ranked[0]?.answer ?? false}`); + } + for (const f of ranked.slice(0, 12)) { + console.log(` ${f.answer ? '*' : ' '} ${pct(f.share).padStart(6)} ${String(f.chars).padStart(6)} ${f.path}`); + } + if (ranked.length > 12) console.log(` … ${ranked.length - 12} more files`); +}