test(agent-eval): RUNS knob + explore envelope-share view for new-vs-baseline A/B (CG-15)
ab-new-vs-baseline.sh now builds and indexes once per arm and runs the task RUNS times (default 1), so the >=2-runs-per-arm rule costs one build instead of N. Both arms run with CODEGRAPH_NO_PROMPT_HOOK=1 — the machine's ambient front-load hook resolves to whatever is in dist/, a second uncontrolled channel that confounds the tool-call counts — and point explore's CG-4 diagnostic at a per-arm sidecar. parse-run.mjs gains --envelope/--answer: the per-file share of the explore source envelope, parsed from the rendered markdown so it works on ANY build. The CG-4 sidecar only exists post-CG-4, so it cannot measure the baseline arm; this is the only view that measures both arms the same way. Folded into parse-run.mjs rather than added as a new script on purpose: a new file named after explore's budget scores into the self-query fixture's own corpus and moved its answer share 59.9%% -> 47.9%%. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
co-authored by
Claude Opus 5
Claude Opus 5
parent
1d9206d2d0
commit
48a2309b92
@@ -21,7 +21,19 @@
|
||||
# <indexed-repo> a repo with a .codegraph index (copied per arm)
|
||||
# "<task>" an implementation task, e.g. "Add X to Y and wire it through"
|
||||
# [baseline-ref] git ref for the BEFORE build (default: HEAD~1)
|
||||
# Env: AGENT_EVAL_OUT (default: /tmp/ab-new-vs-baseline)
|
||||
# Env:
|
||||
# AGENT_EVAL_OUT output dir (default: /tmp/ab-new-vs-baseline)
|
||||
# RUNS runs per arm (default 1). Run-to-run variance is large —
|
||||
# use >=2 and report the range, never a single run. Both arms
|
||||
# build/index ONCE and then run RUNS times, so raising this is
|
||||
# far cheaper than re-invoking the script.
|
||||
# MODEL / EFFORT default sonnet / high. Never raise without a reason: sonnet
|
||||
# is the deliberate floor model (see CLAUDE.md).
|
||||
#
|
||||
# Both arms run with CODEGRAPH_NO_PROMPT_HOOK=1: the machine's ambient
|
||||
# UserPromptSubmit front-load hook resolves to whichever build is currently in
|
||||
# dist/, so leaving it on injects context through a second, uncontrolled channel
|
||||
# and confounds the tool-call counts this script exists to compare.
|
||||
set -uo pipefail
|
||||
|
||||
TARGET="${1:?usage: ab-new-vs-baseline.sh <indexed-repo> \"<task>\" [baseline-ref]}"
|
||||
@@ -67,18 +79,27 @@ prewarm() { # target — spawn a persistent daemon (current $BIN) and wait for i
|
||||
&& echo " daemon warm: $1" || echo " WARN: daemon never bound for $1 (arm may run without codegraph)"
|
||||
}
|
||||
|
||||
run_arm() { # label, target-copy
|
||||
run_arm() { # label, target-copy — runs the task $RUNS times against one build
|
||||
local label="$1" tgt="$2" c="$OUT/mcp-$1.json"
|
||||
# Connect to the pre-warmed daemon; skip the startup re-exec for a fast attach.
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
|
||||
prewarm "$tgt"
|
||||
# CODEGRAPH_EXPLORE_DEBUG points explore's per-file allocation diagnostic at a
|
||||
# sidecar (no-op on builds predating it; never perturbs the response).
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_EXPLORE_DEBUG=%s","node","%s","serve","--mcp","--path","%s"]}}}' \
|
||||
"$OUT/explore-$label.jsonl" "$BIN" "$tgt" > "$c"
|
||||
rm -f "$OUT/explore-$label.jsonl"
|
||||
echo "############## ARM [$label] ##############"
|
||||
( cd "$tgt" && claude -p "$TASK" \
|
||||
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
|
||||
</dev/null > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
|
||||
node "$PARSE" "$OUT/run-$label.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label.jsonl)"
|
||||
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
||||
for i in $(seq 1 "${RUNS:-1}"); do
|
||||
# Re-warm per run: the previous run's daemon is killed below, and a cold
|
||||
# attach is exactly the failure this pre-warm exists to prevent.
|
||||
prewarm "$tgt"
|
||||
( cd "$tgt" && CODEGRAPH_NO_PROMPT_HOOK=1 claude -p "$TASK" \
|
||||
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
|
||||
</dev/null > "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" )
|
||||
echo "-- run $i --"
|
||||
node "$PARSE" "$OUT/run-$label-$i.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label-$i.jsonl)"
|
||||
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
||||
done
|
||||
echo
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,33 @@
|
||||
#!/usr/bin/env node
|
||||
// Parse a Claude Code stream-json run log: tool-call sequence + token usage.
|
||||
//
|
||||
// With --envelope it also reports how the codegraph_explore responses the agent
|
||||
// received were DIVIDED across files — the per-file share of the source envelope.
|
||||
// That view is parsed out of the rendered markdown rather than the CG-4
|
||||
// diagnostic sidecar, so it works on ANY build (the sidecar only exists post-CG-4)
|
||||
// and is therefore the only way to measure both arms of a new-vs-baseline A/B the
|
||||
// same way. `--answer <glob>` (repeatable) marks the files that actually answer
|
||||
// the question, and the summary reports their combined share.
|
||||
//
|
||||
// Usage: parse-run.mjs <run.jsonl> [--envelope] [--answer <glob>]...
|
||||
import { readFileSync } from 'fs';
|
||||
const file = process.argv[2];
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const answerGlobs = [];
|
||||
let file = null;
|
||||
let wantEnvelope = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--envelope') wantEnvelope = true;
|
||||
else if (argv[i] === '--answer') { answerGlobs.push(argv[++i]); wantEnvelope = true; }
|
||||
else if (!argv[i].startsWith('--') && file === null) file = argv[i];
|
||||
}
|
||||
const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
|
||||
|
||||
const toolCalls = [];
|
||||
let result = null;
|
||||
let initTools = null;
|
||||
const exploreQueries = new Map(); // tool_use id -> query
|
||||
const exploreTexts = []; // response text, in call order
|
||||
|
||||
for (const line of lines) {
|
||||
let ev;
|
||||
@@ -23,6 +44,16 @@ for (const line of lines) {
|
||||
else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`;
|
||||
else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`;
|
||||
toolCalls.push(`${block.name}${detail}`);
|
||||
if (/codegraph_explore/.test(block.name)) exploreQueries.set(block.id, block.input?.query ?? '');
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ev.type === 'user' && ev.message?.content) {
|
||||
for (const block of ev.message.content) {
|
||||
if (block.type === 'tool_result' && exploreQueries.has(block.tool_use_id)) {
|
||||
exploreTexts.push(typeof block.content === 'string'
|
||||
? block.content
|
||||
: (block.content ?? []).filter(c => c.type === 'text').map(c => c.text).join('\n'));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,3 +74,53 @@ if (result) {
|
||||
console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`);
|
||||
console.log(` tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`);
|
||||
}
|
||||
|
||||
// ---- envelope share (opt-in) ------------------------------------------------
|
||||
|
||||
if (wantEnvelope) {
|
||||
// `tools/cache/**` -> /^tools\/cache\/.*$/ . Same semantics as probe-allocation.
|
||||
// The `**` sentinel is written as an escape, never a literal NUL byte — a raw
|
||||
// one makes git treat this whole script as binary and costs every future diff.
|
||||
const glob2re = (glob) => {
|
||||
const S = '\u0000';
|
||||
const body = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
||||
.replace(/\*\*/g, S).replace(/\*/g, '[^/]*').replaceAll(S, '.*');
|
||||
return new RegExp(`^${body}$`);
|
||||
};
|
||||
const answerRes = answerGlobs.map(glob2re);
|
||||
const isAnswer = (p) => answerRes.some(re => re.test(p));
|
||||
|
||||
// Each rendered file section starts with **`path`** — its bytes run to the next
|
||||
// such header (or to the trailing guidance quote). Share is over the sum of the
|
||||
// sections, i.e. of the source envelope the allocator divides.
|
||||
const pooled = new Map();
|
||||
let envelope = 0;
|
||||
for (const text of exploreTexts) {
|
||||
const re = /^\*\*`([^`]+)`\*\*/gm;
|
||||
const marks = [];
|
||||
let m;
|
||||
while ((m = re.exec(text)) !== null) marks.push({ path: m[1], at: m.index });
|
||||
if (!marks.length) continue;
|
||||
const tail = text.indexOf('\n> ', marks[marks.length - 1].at);
|
||||
const end = tail === -1 ? text.length : tail;
|
||||
marks.forEach((mark, i) => {
|
||||
const chars = (i + 1 < marks.length ? marks[i + 1].at : end) - mark.at;
|
||||
pooled.set(mark.path, (pooled.get(mark.path) ?? 0) + chars);
|
||||
envelope += chars;
|
||||
});
|
||||
}
|
||||
const ranked = [...pooled.entries()]
|
||||
.map(([path, chars]) => ({ path, chars, share: envelope ? chars / envelope : 0, answer: isAnswer(path) }))
|
||||
.sort((a, b) => b.chars - a.chars);
|
||||
const answerChars = ranked.filter(r => r.answer).reduce((s, r) => s + r.chars, 0);
|
||||
const pct = (f) => `${(f * 100).toFixed(1)}%`;
|
||||
|
||||
console.log(`\nExplore envelope: ${envelope.toLocaleString('en-US')} chars over ${exploreTexts.length} response(s)`);
|
||||
if (answerGlobs.length) {
|
||||
console.log(` answer-set share: ${pct(envelope ? answerChars / envelope : 0)} | top file answers: ${ranked[0]?.answer ?? false}`);
|
||||
}
|
||||
for (const f of ranked.slice(0, 12)) {
|
||||
console.log(` ${f.answer ? '*' : ' '} ${pct(f.share).padStart(6)} ${String(f.chars).padStart(6)} ${f.path}`);
|
||||
}
|
||||
if (ranked.length > 12) console.log(` … ${ranked.length - 12} more files`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user