test(agent-eval): stop the arms reaching codegraph through the shell (CG-7)
The without-arm had no MCP server but still had Bash, and the target repo carries the .codegraph/ index the with-arm needs. Agents found it: 14 of 15 without-arm runs in a 7-repo pass ran `codegraph explore` through Bash, one of them via `ls .codegraph && codegraph explore ...`. That arm was measuring codegraph-over-CLI, not codegraph-absent, so every without-arm number it produced was wrong. It bit the with-arm too -- output arriving through Bash is attributed to Bash, understating what codegraph itself occupies (1 of 15 runs). Both arms now run on a PATH where the CLI is hidden, so the MCP server is the only way to reach codegraph and stays the single variable. The binary shares a directory with tools the run needs -- claude itself sits next to it -- so the directory is substituted in place by one of symlinks to every entry except codegraph, preserving PATH order and precedence. The run aborts if claude or node did not survive the substitution. Prevention alone would fail silently the next time the CLI lands somewhere new, so parse-run.mjs flags any Bash command naming codegraph and parse-bench-readme drops contaminated without-arm runs from the aggregate (CG_INCLUDE_CONTAMINATED=1 keeps them). CG_ARMS re-runs one arm without redoing the other.
This commit is contained in:
@@ -42,7 +42,7 @@ function parse(dir, label) {
|
||||
const o = s.occupancy;
|
||||
return {
|
||||
dur: s.dur, tools: s.tools, reads: s.reads, grep: s.grep, cg: s.cg,
|
||||
bash: s.counts.Bash || 0,
|
||||
bash: s.counts.Bash || 0, cliCalls: s.cliCalls,
|
||||
tokens: s.processed, cost: s.cost, raced: s.raced, turns: s.turns,
|
||||
segments: files.length,
|
||||
ctx: o.ctxFinal,
|
||||
@@ -67,7 +67,11 @@ const pct = (w, wo) => wo > 0 ? Math.round((1 - w / wo) * 100) : 0;
|
||||
// race, not steady-state value. `CG_INCLUDE_RACED=1` keeps them (to see the raw
|
||||
// distribution). The WITHOUT arm has no MCP, so it's never raced.
|
||||
const includeRaced = process.env.CG_INCLUDE_RACED === '1';
|
||||
// A without-arm run that shelled out to the codegraph CLI measured
|
||||
// codegraph-over-CLI, not codegraph-absent. Drop it unless asked otherwise.
|
||||
const includeContaminated = process.env.CG_INCLUDE_CONTAMINATED === '1';
|
||||
const rows = [];
|
||||
let contaminated = 0;
|
||||
for (const repo of REPOS) {
|
||||
const dir = join(ROOT, repo);
|
||||
const runDirs = existsSync(dir) ? readdirSync(dir).filter(d => /^run\d+$/.test(d)).sort() : [];
|
||||
@@ -75,10 +79,15 @@ for (const repo of REPOS) {
|
||||
for (const rd of runDirs) {
|
||||
const w = parse(join(dir, rd), 'headless-with');
|
||||
if (w) { if (w.raced && !includeRaced) racedExcluded++; else W.push(w); }
|
||||
const wo = parse(join(dir, rd), 'headless-without'); if (wo) WO.push(wo);
|
||||
const wo = parse(join(dir, rd), 'headless-without');
|
||||
if (wo) {
|
||||
if (wo.cliCalls && !includeContaminated) { contaminated++; console.error(`[excluded] ${repo}/${rd} without-arm ran the codegraph CLI ${wo.cliCalls}x`); }
|
||||
else WO.push(wo);
|
||||
}
|
||||
}
|
||||
rows.push({ repo, W, WO, racedExcluded });
|
||||
}
|
||||
if (contaminated) console.error(`[excluded] ${contaminated} contaminated without-arm run(s); CG_INCLUDE_CONTAMINATED=1 keeps them\n`);
|
||||
|
||||
// ---- Table 1: the existing throughput view. --------------------------------
|
||||
console.log('repo n(w/wo) time WITH→WITHOUT tools W→WO tokens W→WO (saved) cost W→WO (saved)');
|
||||
|
||||
Reference in New Issue
Block a user