test(agent-eval): report all three feedback metrics per arm, side by side (CG-11)
The three metrics existed but only run-all.sh printed them, one block per run. ab-new-vs-baseline.sh — the harness that actually isolates a retrieval change, both arms codegraph-on — grepped its parse output down to `by type` and `Result`, so occupancy, sufficiency and allocation never reached the maintainer running the A/B they were built for. Both harnesses now print the three blocks under every run and end with one compare-arms.mjs table: median [min–max] per arm across RUNS, sufficiency pooled (it is per-CALL, so median-of-run-percentages would weight a 1-call run like a 5-call one), allocation pooled by bytes and per run. The table is "did it move?"; the per-run blocks stay the "why?" — only they name the query that fell short and the file nothing cited. It reproduces the recorded CG-22 express result off logs already on disk: baseline 3/6 calls in the `Read a file we returned` bucket at 82.0%, new 0/5 at 96.9%. parse-bench-readme.mjs gets the same two metrics as a with-arm table, so the CG-13 campaign aggregates all three rather than occupancy alone. Also folds the CLI-block shim into no-cli-shim.sh and gives it to ab-new-vs-baseline.sh. There it is not a with/without leak but an attribution one, and it breaks all three metrics at once: output arriving through Bash is charged to Bash in the occupancy table, and an explore issued through the CLI is not a tool call at all, so it never reaches the sufficiency classifier or the allocation parse. The run silently drops calls from every number. The daemon pre-warm and the model policy are untouched. Validated on one live gin arm (2 explores, 0 Read, all three blocks + table) and against the cg22/cg15 and ab-readme logs. Selftest 68/68.
This commit is contained in:
@@ -17,7 +17,7 @@
|
||||
// Usage: node parse-bench-readme.mjs [/tmp/ab-readme]
|
||||
import { existsSync, readdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { parseSession } from './parse-run.mjs';
|
||||
import { parseSession, SUFFICIENCY } from './parse-run.mjs';
|
||||
|
||||
const ROOT = process.argv[2] || '/tmp/ab-readme';
|
||||
const REPOS = ['vscode', 'excalidraw', 'django', 'tokio', 'okhttp', 'gin', 'alamofire'];
|
||||
@@ -55,6 +55,16 @@ function parse(dir, label) {
|
||||
occShareCtx: o.ctxFinal > 0 ? ((o.residual.codegraph + o.residualFileAccess) / o.ctxFinal) * 100 : 0,
|
||||
occShareWin: ((o.residual.codegraph + o.residualFileAccess) / o.windowTokens) * 100,
|
||||
window: o.windowTokens,
|
||||
// The other two feedback metrics, carried per run so the campaign can pool
|
||||
// them. Both are with-arm-only in practice — a without-arm makes no explore
|
||||
// calls, so it has nothing to be sufficient about and no bytes to allocate.
|
||||
suffAnswered: s.sufficiency.answered,
|
||||
suffCounts: s.sufficiency.counts,
|
||||
// Byte-weighted, so a run with no explore contributes NOTHING rather than a
|
||||
// zero; a zero would drag a repo toward "wasteful" for never spending a byte.
|
||||
allocUsed: s.allocation.envelope ? s.allocation.used : 0,
|
||||
allocEnvelope: s.allocation.envelope,
|
||||
allocCalls: s.allocation.calls.length,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -161,3 +171,50 @@ if (!anyMulti) {
|
||||
`follow-ups (see run-all.sh) to measure the regime this metric is actually about.`
|
||||
);
|
||||
}
|
||||
|
||||
// ---- Table 3: sufficiency + allocation, WITH arm only. ---------------------
|
||||
// Occupancy says what a response COST; these two say whether it was enough and
|
||||
// whether it spent its bytes on the right files. A campaign that reports only
|
||||
// occupancy cannot tell a tighter response from a worse one.
|
||||
//
|
||||
// Pooled per repo, not median-of-runs: both are per-CALL quantities (sufficiency
|
||||
// counts explores, allocation weights by bytes), and a repo contributes 2-15
|
||||
// calls across its runs. Median-of-run-percentages would weight a 1-call run the
|
||||
// same as a 5-call one.
|
||||
console.log(`\n\nEXPLORE SUFFICIENCY + ALLOCATION EFFICIENCY — with-arm only, pooled over runs`);
|
||||
console.log(`(sufficiency = what the agent did NEXT · allocation = share of returned bytes the answer cited)\n`);
|
||||
console.log('repo calls again read-ret read-miss grep MOVED ON alloc eff envelope');
|
||||
const totals = { answered: 0, counts: Object.fromEntries(SUFFICIENCY.map(([k]) => [k, 0])), used: 0, env: 0, calls: 0 };
|
||||
for (const { repo, W } of rows) {
|
||||
if (!W.length) { console.log(`${repo.padEnd(11)} (no with-arm runs)`); continue; }
|
||||
const answered = W.reduce((s, r) => s + r.suffAnswered, 0);
|
||||
const cnt = (k) => W.reduce((s, r) => s + r.suffCounts[k], 0);
|
||||
const env = W.reduce((s, r) => s + r.allocEnvelope, 0);
|
||||
const used = W.reduce((s, r) => s + r.allocUsed, 0);
|
||||
totals.answered += answered; totals.used += used; totals.env += env;
|
||||
totals.calls += W.reduce((s, r) => s + r.allocCalls, 0);
|
||||
for (const [k] of SUFFICIENCY) totals.counts[k] += cnt(k);
|
||||
const cell = (k) => (answered ? `${cnt(k)} ${Math.round((cnt(k) / answered) * 100)}%` : '—').padEnd(9);
|
||||
console.log(
|
||||
`${repo.padEnd(11)} ${String(answered).padEnd(7)} ` +
|
||||
`${cell('explore_again')}${cell('read_returned')}${cell('read_missed')}${cell('search')}` +
|
||||
`${(answered ? `${cnt('sufficient')} ${Math.round((cnt('sufficient') / answered) * 100)}%` : '—').padEnd(13)}` +
|
||||
`${(env ? `${((used / env) * 100).toFixed(1)}%` : '—').padEnd(12)}${fmtTok(env)}`
|
||||
);
|
||||
}
|
||||
const tp = (k) => totals.answered ? `${totals.counts[k]} (${Math.round((totals.counts[k] / totals.answered) * 100)}%)` : '—';
|
||||
console.log(
|
||||
`\nPOOLED (${totals.answered} answered explore calls): ` +
|
||||
SUFFICIENCY.map(([k, label]) => `${label} ${tp(k)}`).join(' · ')
|
||||
);
|
||||
console.log(
|
||||
`POOLED allocation efficiency: ${totals.env ? ((totals.used / totals.env) * 100).toFixed(1) + '%' : '—'} ` +
|
||||
`over ${totals.calls} calls / ${fmtTok(totals.env)} chars`
|
||||
);
|
||||
console.log(
|
||||
`\nHOW TO READ: "read-ret" (Read a file we RETURNED) is an allocation miss — right file,\n` +
|
||||
`wrong bytes; "read-miss" and "grep" are recall misses. "again" is ambiguous by construction.\n` +
|
||||
`Allocation efficiency is RELATIVE — attribution is by citation, so it compares BUILDS on the\n` +
|
||||
`same questions and is not a claim that codegraph wasted the remainder. Full guidance:\n` +
|
||||
`docs/benchmarks/agent-eval-feedback-metrics.md`
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user