test(agent-eval): stop the arms reaching codegraph through the shell (CG-7)

The without-arm had no MCP server but still had Bash, and the target repo carries
the .codegraph/ index the with-arm needs. Agents found it: 14 of 15 without-arm
runs in a 7-repo pass ran `codegraph explore` through Bash, one of them via
`ls .codegraph && codegraph explore ...`. That arm was measuring
codegraph-over-CLI, not codegraph-absent, so every without-arm number it produced
was wrong. It bit the with-arm too -- output arriving through Bash is attributed
to Bash, understating what codegraph itself occupies (1 of 15 runs).

Both arms now run on a PATH where the CLI is hidden, so the MCP server is the
only way to reach codegraph and stays the single variable. The binary shares a
directory with tools the run needs -- claude itself sits next to it -- so the
directory is substituted in place by one of symlinks to every entry except
codegraph, preserving PATH order and precedence. The run aborts if claude or node
did not survive the substitution.

Prevention alone would fail silently the next time the CLI lands somewhere new,
so parse-run.mjs flags any Bash command naming codegraph and parse-bench-readme
drops contaminated without-arm runs from the aggregate (CG_INCLUDE_CONTAMINATED=1
keeps them). CG_ARMS re-runs one arm without redoing the other.
This commit is contained in:
Colby McHenry
2026-08-04 15:30:28 -05:00
parent 257a7b7327
commit d3c01ce8ed
3 changed files with 66 additions and 8 deletions
+11 -2
View File
@@ -42,7 +42,7 @@ function parse(dir, label) {
const o = s.occupancy;
return {
dur: s.dur, tools: s.tools, reads: s.reads, grep: s.grep, cg: s.cg,
bash: s.counts.Bash || 0,
bash: s.counts.Bash || 0, cliCalls: s.cliCalls,
tokens: s.processed, cost: s.cost, raced: s.raced, turns: s.turns,
segments: files.length,
ctx: o.ctxFinal,
@@ -67,7 +67,11 @@ const pct = (w, wo) => wo > 0 ? Math.round((1 - w / wo) * 100) : 0;
// race, not steady-state value. `CG_INCLUDE_RACED=1` keeps them (to see the raw
// distribution). The WITHOUT arm has no MCP, so it's never raced.
const includeRaced = process.env.CG_INCLUDE_RACED === '1';
// A without-arm run that shelled out to the codegraph CLI measured
// codegraph-over-CLI, not codegraph-absent. Drop it unless asked otherwise.
const includeContaminated = process.env.CG_INCLUDE_CONTAMINATED === '1';
const rows = [];
let contaminated = 0;
for (const repo of REPOS) {
const dir = join(ROOT, repo);
const runDirs = existsSync(dir) ? readdirSync(dir).filter(d => /^run\d+$/.test(d)).sort() : [];
@@ -75,10 +79,15 @@ for (const repo of REPOS) {
for (const rd of runDirs) {
const w = parse(join(dir, rd), 'headless-with');
if (w) { if (w.raced && !includeRaced) racedExcluded++; else W.push(w); }
const wo = parse(join(dir, rd), 'headless-without'); if (wo) WO.push(wo);
const wo = parse(join(dir, rd), 'headless-without');
if (wo) {
if (wo.cliCalls && !includeContaminated) { contaminated++; console.error(`[excluded] ${repo}/${rd} without-arm ran the codegraph CLI ${wo.cliCalls}x`); }
else WO.push(wo);
}
}
rows.push({ repo, W, WO, racedExcluded });
}
if (contaminated) console.error(`[excluded] ${contaminated} contaminated without-arm run(s); CG_INCLUDE_CONTAMINATED=1 keeps them\n`);
// ---- Table 1: the existing throughput view. --------------------------------
console.log('repo n(w/wo) time WITH→WITHOUT tools W→WO tokens W→WO (saved) cost W→WO (saved)');