test(agent-eval): report all three feedback metrics per arm, side by side (CG-11)

The three metrics existed but only run-all.sh printed them, one block per
run. ab-new-vs-baseline.sh — the harness that actually isolates a retrieval
change, both arms codegraph-on — grepped its parse output down to `by type`
and `Result`, so occupancy, sufficiency and allocation never reached the
maintainer running the A/B they were built for.

Both harnesses now print the three blocks under every run and end with one
compare-arms.mjs table: median [min–max] per arm across RUNS, sufficiency
pooled (it is per-CALL, so median-of-run-percentages would weight a 1-call
run like a 5-call one), allocation pooled by bytes and per run. The table is
"did it move?"; the per-run blocks stay the "why?" — only they name the query
that fell short and the file nothing cited. It reproduces the recorded CG-22
express result off logs already on disk: baseline 3/6 calls in the
`Read a file we returned` bucket at 82.0%, new 0/5 at 96.9%.

parse-bench-readme.mjs gets the same two metrics as a with-arm table, so the
CG-13 campaign aggregates all three rather than occupancy alone.

Also folds the CLI-block shim into no-cli-shim.sh and gives it to
ab-new-vs-baseline.sh. There it is not a with/without leak but an attribution
one, and it breaks all three metrics at once: output arriving through Bash is
charged to Bash in the occupancy table, and an explore issued through the CLI
is not a tool call at all, so it never reaches the sufficiency classifier or
the allocation parse. The run silently drops calls from every number.

The daemon pre-warm and the model policy are untouched.

Validated on one live gin arm (2 explores, 0 Read, all three blocks + table)
and against the cg22/cg15 and ab-readme logs. Selftest 68/68.
This commit is contained in:
Colby McHenry
2026-08-05 00:58:54 -05:00
parent fa15d1046a
commit 3e8922dfad
6 changed files with 440 additions and 81 deletions
+58 -1
View File
@@ -17,7 +17,7 @@
// Usage: node parse-bench-readme.mjs [/tmp/ab-readme]
import { existsSync, readdirSync } from 'fs';
import { join } from 'path';
import { parseSession } from './parse-run.mjs';
import { parseSession, SUFFICIENCY } from './parse-run.mjs';
const ROOT = process.argv[2] || '/tmp/ab-readme';
const REPOS = ['vscode', 'excalidraw', 'django', 'tokio', 'okhttp', 'gin', 'alamofire'];
@@ -55,6 +55,16 @@ function parse(dir, label) {
occShareCtx: o.ctxFinal > 0 ? ((o.residual.codegraph + o.residualFileAccess) / o.ctxFinal) * 100 : 0,
occShareWin: ((o.residual.codegraph + o.residualFileAccess) / o.windowTokens) * 100,
window: o.windowTokens,
// The other two feedback metrics, carried per run so the campaign can pool
// them. Both are with-arm-only in practice — a without-arm makes no explore
// calls, so it has nothing to be sufficient about and no bytes to allocate.
suffAnswered: s.sufficiency.answered,
suffCounts: s.sufficiency.counts,
// Byte-weighted, so a run with no explore contributes NOTHING rather than a
// zero; a zero would drag a repo toward "wasteful" for never spending a byte.
allocUsed: s.allocation.envelope ? s.allocation.used : 0,
allocEnvelope: s.allocation.envelope,
allocCalls: s.allocation.calls.length,
};
}
@@ -161,3 +171,50 @@ if (!anyMulti) {
`follow-ups (see run-all.sh) to measure the regime this metric is actually about.`
);
}
// ---- Table 3: sufficiency + allocation, WITH arm only. ---------------------
// Occupancy says what a response COST; these two say whether it was enough and
// whether it spent its bytes on the right files. A campaign that reports only
// occupancy cannot tell a tighter response from a worse one.
//
// Pooled per repo, not median-of-runs: both are per-CALL quantities (sufficiency
// counts explores, allocation weights by bytes), and a repo contributes 2-15
// calls across its runs. Median-of-run-percentages would weight a 1-call run the
// same as a 5-call one.
console.log(`\n\nEXPLORE SUFFICIENCY + ALLOCATION EFFICIENCY — with-arm only, pooled over runs`);
console.log(`(sufficiency = what the agent did NEXT · allocation = share of returned bytes the answer cited)\n`);
console.log('repo calls again read-ret read-miss grep MOVED ON alloc eff envelope');
const totals = { answered: 0, counts: Object.fromEntries(SUFFICIENCY.map(([k]) => [k, 0])), used: 0, env: 0, calls: 0 };
for (const { repo, W } of rows) {
if (!W.length) { console.log(`${repo.padEnd(11)} (no with-arm runs)`); continue; }
const answered = W.reduce((s, r) => s + r.suffAnswered, 0);
const cnt = (k) => W.reduce((s, r) => s + r.suffCounts[k], 0);
const env = W.reduce((s, r) => s + r.allocEnvelope, 0);
const used = W.reduce((s, r) => s + r.allocUsed, 0);
totals.answered += answered; totals.used += used; totals.env += env;
totals.calls += W.reduce((s, r) => s + r.allocCalls, 0);
for (const [k] of SUFFICIENCY) totals.counts[k] += cnt(k);
const cell = (k) => (answered ? `${cnt(k)} ${Math.round((cnt(k) / answered) * 100)}%` : '—').padEnd(9);
console.log(
`${repo.padEnd(11)} ${String(answered).padEnd(7)} ` +
`${cell('explore_again')}${cell('read_returned')}${cell('read_missed')}${cell('search')}` +
`${(answered ? `${cnt('sufficient')} ${Math.round((cnt('sufficient') / answered) * 100)}%` : '—').padEnd(13)}` +
`${(env ? `${((used / env) * 100).toFixed(1)}%` : '—').padEnd(12)}${fmtTok(env)}`
);
}
const tp = (k) => totals.answered ? `${totals.counts[k]} (${Math.round((totals.counts[k] / totals.answered) * 100)}%)` : '—';
console.log(
`\nPOOLED (${totals.answered} answered explore calls): ` +
SUFFICIENCY.map(([k, label]) => `${label} ${tp(k)}`).join(' · ')
);
console.log(
`POOLED allocation efficiency: ${totals.env ? ((totals.used / totals.env) * 100).toFixed(1) + '%' : '—'} ` +
`over ${totals.calls} calls / ${fmtTok(totals.env)} chars`
);
console.log(
`\nHOW TO READ: "read-ret" (Read a file we RETURNED) is an allocation miss — right file,\n` +
`wrong bytes; "read-miss" and "grep" are recall misses. "again" is ambiguous by construction.\n` +
`Allocation efficiency is RELATIVE — attribution is by citation, so it compares BUILDS on the\n` +
`same questions and is not a claim that codegraph wasted the remainder. Full guidance:\n` +
`docs/benchmarks/agent-eval-feedback-metrics.md`
);