From 3e8922dfad19d3ff1b435f53cd1e22815304d1d1 Mon Sep 17 00:00:00 2001
From: Colby McHenry <me@colbymchenry.com>
Date: Wed, 5 Aug 2026 00:58:42 -0500
Subject: [PATCH] test(agent-eval): report all three feedback metrics per arm,
 side by side (CG-11)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The three metrics existed but only run-all.sh printed them, one block per
run. ab-new-vs-baseline.sh — the harness that actually isolates a retrieval
change, both arms codegraph-on — grepped its parse output down to `by type`
and `Result`, so occupancy, sufficiency and allocation never reached the
maintainer running the A/B they were built for.

Both harnesses now print the three blocks under every run and end with one
compare-arms.mjs table: median [min–max] per arm across RUNS, sufficiency
pooled (it is per-CALL, so median-of-run-percentages would weight a 1-call
run like a 5-call one), allocation pooled by bytes and per run. The table is
"did it move?"; the per-run blocks stay the "why?" — only they name the query
that fell short and the file nothing cited. It reproduces the recorded CG-22
express result off logs already on disk: baseline 3/6 calls in the
`Read a file we returned` bucket at 82.0%, new 0/5 at 96.9%.

parse-bench-readme.mjs gets the same two metrics as a with-arm table, so the
CG-13 campaign aggregates all three rather than occupancy alone.

Also folds the CLI-block shim into no-cli-shim.sh and gives it to
ab-new-vs-baseline.sh. There it is not a with/without leak but an attribution
one, and it breaks all three metrics at once: output arriving through Bash is
charged to Bash in the occupancy table, and an explore issued through the CLI
is not a tool call at all, so it never reaches the sufficiency classifier or
the allocation parse. The run silently drops calls from every number.

The daemon pre-warm and the model policy are untouched.

Validated on one live gin arm (2 explores, 0 Read, all three blocks + table)
and against the cg22/cg15 and ab-readme logs. Selftest 68/68.
---
 scripts/agent-eval/ab-new-vs-baseline.sh  |  54 +++++-
 scripts/agent-eval/compare-arms.mjs       | 209 ++++++++++++++++++++++
 scripts/agent-eval/no-cli-shim.sh         |  96 ++++++++++
 scripts/agent-eval/parse-bench-readme.mjs |  59 +++++-
 scripts/agent-eval/parse-run.mjs          |  14 +-
 scripts/agent-eval/run-all.sh             |  89 +++------
 6 files changed, 440 insertions(+), 81 deletions(-)
 create mode 100644 scripts/agent-eval/compare-arms.mjs
 create mode 100644 scripts/agent-eval/no-cli-shim.sh

diff --git a/scripts/agent-eval/ab-new-vs-baseline.sh b/scripts/agent-eval/ab-new-vs-baseline.sh
index ebc353c..88f5b5a 100755
--- a/scripts/agent-eval/ab-new-vs-baseline.sh
+++ b/scripts/agent-eval/ab-new-vs-baseline.sh
@@ -1,9 +1,27 @@
 #!/usr/bin/env bash
 # A/B a codegraph retrieval/steering change: the NEW build (current HEAD) vs a
 # BASELINE build (a git ref) — BOTH with codegraph attached — on the same
-# implementation task, measuring how many Read vs codegraph calls the agent
-# makes. ISOLATES the change (unlike run-all.sh's with-vs-without). The agent
-# works on a throwaway copy of the target, so your repos are never touched.
+# implementation task. ISOLATES the change (unlike run-all.sh's
+# with-vs-without). The agent works on a throwaway copy of the target, so your
+# repos are never touched.
+#
+# Each run reports the tool mix AND the three feedback metrics; compare-arms.mjs
+# then puts both arms side by side:
+#   residual context occupancy (CG-7)  window still held by the arm's retrieval
+#   explore sufficiency        (CG-8)  was the response ENOUGH — the agent's next act
+#   allocation efficiency      (CG-9)  share of returned bytes the answer cited
+# This is the harness those metrics were built for: both arms are codegraph-on,
+# so every one of them is measuring the retrieval change rather than adoption.
+# docs/benchmarks/agent-eval-feedback-metrics.md is the entry point; the three
+# per-metric docs it links carry the caveats — in particular that allocation
+# efficiency is RELATIVE (attribution is by citation), which is exactly why
+# same-question new-vs-baseline is the comparison it is valid for.
+#
+# Both arms also run with the codegraph CLI blocked (no-cli-shim.sh). Here that
+# is not a with/without leak but an ATTRIBUTION one: an explore issued through
+# Bash is charged to Bash in the occupancy table and never reaches the
+# sufficiency or allocation parse at all, so a run that shells out silently
+# drops calls from all three metrics.
 #
 # Reliable attach (works even when this is itself run nested inside a Claude
 # session): each arm PRE-WARMS a persistent codegraph daemon for its target so
@@ -39,10 +57,11 @@ set -uo pipefail
 TARGET="${1:?usage: ab-new-vs-baseline.sh <indexed-repo> \"<task>\" [baseline-ref]}"
 TASK="${2:?task required}"
 BASE_REF="${3:-HEAD~1}"
-ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
+HARNESS="$(cd "$(dirname "$0")" && pwd)"
+ENGINE="$(cd "$HARNESS/../.." && pwd)"
 BIN="$ENGINE/dist/bin/codegraph.js"
 OUT="${AGENT_EVAL_OUT:-/tmp/ab-new-vs-baseline}"
-PARSE="$ENGINE/scripts/agent-eval/parse-run.mjs"
+PARSE="$HARNESS/parse-run.mjs"
 
 command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
 [ -d "$TARGET/.codegraph" ] || { echo "target not indexed: run 'codegraph init $TARGET' first"; exit 1; }
@@ -63,6 +82,12 @@ cleanup() {
 trap cleanup EXIT INT TERM
 
 mkdir -p "$OUT"
+
+# Sanitized PATH + the absolute-path block, shared with run-all.sh. Sets
+# $ARM_PATH and $ARM_SETTINGS; aborts if either layer fails its own probe.
+. "$HARNESS/no-cli-shim.sh"
+cg_no_cli_setup "$OUT" || exit 1
+
 echo "###### engine=$ENGINE  baseline=$BASE_REF"
 echo "###### changed: $(echo "$CHANGED" | tr '\n' ' ')"
 echo "###### target=$TARGET"
@@ -94,12 +119,16 @@ run_arm() { # label, target-copy — runs the task $RUNS times against one build
     # Re-warm per run: the previous run's daemon is killed below, and a cold
     # attach is exactly the failure this pre-warm exists to prevent.
     prewarm "$tgt"
-    ( cd "$tgt" && CODEGRAPH_NO_PROMPT_HOOK=1 claude -p "$TASK" \
+    ( cd "$tgt" && PATH="$ARM_PATH" CODEGRAPH_NO_PROMPT_HOOK=1 claude -p "$TASK" \
         --output-format stream-json --verbose --permission-mode bypassPermissions \
         --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
+        --settings "$ARM_SETTINGS" \
         </dev/null > "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" )
     echo "-- run $i --"
-    node "$PARSE" "$OUT/run-$label-$i.jsonl" 2>&1 | grep -E "by type|Result" || echo "  (parse failed — see $OUT/run-$label-$i.jsonl)"
+    # --brief: tool counts, result, and the three metric blocks, minus the
+    # numbered call transcript (RUNS>=2 is otherwise mostly call listings; the
+    # full sequence is one `parse-run.mjs $OUT/run-$label-$i.jsonl` away).
+    node "$PARSE" --brief "$OUT/run-$label-$i.jsonl" 2>&1 || echo "  (parse failed — see $OUT/run-$label-$i.jsonl)"
     pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
   done
   echo
@@ -121,5 +150,12 @@ done
 node "$BIN" init "$OUT/t-base" >/dev/null 2>&1 && echo "  indexed t-base"
 run_arm baseline "$OUT/t-base"
 
-echo "###### DONE. Compare the [new] vs [baseline] 'by type' counts above"
-echo "###### (especially Read vs mcp__codegraph__*). Full logs in: $OUT"
+# Both arms, all three metrics, one table. The per-run blocks above say WHY a
+# number moved (which query fell short, which file nothing cited); this says
+# whether it moved at all, with the range across RUNS — never read the median
+# of one run.
+node "$HARNESS/compare-arms.mjs" "$OUT" new baseline 2>&1 || true
+
+echo "###### DONE. Read the ARM COMPARISON above first, then the per-run blocks"
+echo "###### for the queries and files behind any number that moved."
+echo "###### Full logs in: $OUT"
diff --git a/scripts/agent-eval/compare-arms.mjs b/scripts/agent-eval/compare-arms.mjs
new file mode 100644
index 0000000..6b13318
--- /dev/null
+++ b/scripts/agent-eval/compare-arms.mjs
@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+// One side-by-side table for the three feedback metrics, across the arms of a
+// single A/B output directory. This is the "did it move?" view — the per-run
+// blocks parse-run.mjs prints are the "why did it move?" view, and both are
+// printed by the harnesses (ab-new-vs-baseline.sh, run-all.sh).
+//
+//   residual context occupancy  how much window the arm's retrieval still holds
+//   explore sufficiency         whether a response was ENOUGH (agent's next act)
+//   allocation efficiency       what share of returned bytes the answer used
+//
+// Usage: compare-arms.mjs <out-dir> <label> [<label> ...]
+//   e.g.  compare-arms.mjs /tmp/ab-new-vs-baseline new baseline
+//         compare-arms.mjs /tmp/agent-eval headless-with headless-without
+//
+// Run discovery handles both shapes the harnesses write, per label:
+//   run-<label>-<i>.jsonl        N independent runs   (ab-new-vs-baseline, RUNS=N)
+//   run-<label>.jsonl + .tN      ONE session, N turns (run-all.sh multi-turn)
+// A `.tN` file is always a resumed SEGMENT of the run it hangs off, never a run
+// of its own — mixing those up would report a three-turn session as three runs
+// and average away the residual the later turns exist to charge.
+import { existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+import { parseSession, SUFFICIENCY } from './parse-run.mjs';
+
+/** Segment files of one run, in turn order: run-X.jsonl, run-X.t2.jsonl, … */
+function segmentsOf(dir, stem) {
+  const first = join(dir, `${stem}.jsonl`);
+  if (!existsSync(first)) return null;
+  const rest = readdirSync(dir)
+    .map((f) => [f, new RegExp(`^${stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.t(\\d+)\\.jsonl$`).exec(f)])
+    .filter(([, m]) => m)
+    .sort((a, b) => Number(a[1][1]) - Number(b[1][1]))
+    .map(([f]) => join(dir, f));
+  return [first, ...rest];
+}
+
+/** Every run of one arm, newest-numbering-first-run order. */
+export function discoverRuns(dir, label) {
+  const session = segmentsOf(dir, `run-${label}`);
+  if (session) return [{ name: label, files: session }];
+  const indexed = readdirSync(dir)
+    .map((f) => new RegExp(`^run-${label.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}-(\\d+)\\.jsonl$`).exec(f))
+    .filter(Boolean)
+    .map((m) => Number(m[1]))
+    .sort((a, b) => a - b);
+  return indexed.map((i) => ({ name: `${label}-${i}`, files: segmentsOf(dir, `run-${label}-${i}`) }));
+}
+
+const median = (xs) => {
+  if (!xs.length) return null;
+  const a = [...xs].sort((x, y) => x - y);
+  const m = a.length >> 1;
+  return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2;
+};
+
+/** The numbers one arm's runs contribute to the table. */
+function measure(run) {
+  const s = parseSession(run.files);
+  const o = s.occupancy;
+  const a = s.allocation;
+  return {
+    name: run.name,
+    ok: s.ok,
+    raced: s.raced,
+    turns: s.turns,
+    dur: s.dur,
+    tools: s.tools,
+    reads: s.reads,
+    grep: s.grep,
+    bash: s.counts.Bash || 0,
+    cg: s.cg,
+    cliCalls: s.cliCalls,
+    cliContaminated: s.cliContaminated,
+    ctx: o.ctxFinal,
+    occCg: o.residual.codegraph,
+    occFile: o.residualFileAccess,
+    // The arm's OWN retrieval residual: codegraph in a with-arm, Read/Grep/Bash
+    // in a without-arm. Comparing these two is the apples-to-apples pair.
+    occSelf: o.residual.codegraph + o.residualFileAccess,
+    occShare: o.ctxFinal > 0 ? ((o.residual.codegraph + o.residualFileAccess) / o.ctxFinal) * 100 : 0,
+    suffAnswered: s.sufficiency.answered,
+    suffCounts: s.sufficiency.counts,
+    suffErrors: s.sufficiency.errors,
+    // Allocation is byte-weighted, so a run with no explore contributes nothing
+    // rather than a zero — a zero would drag the pooled number toward "wasteful"
+    // for a run that never spent a byte.
+    allocUsed: a.envelope ? a.used : null,
+    allocEnvelope: a.envelope || null,
+    allocCalls: a.calls.length,
+  };
+}
+
+/** median [min–max] over runs; the range is the point — never quote one run. */
+function span(runs, pick, fmt = (x) => String(Math.round(x))) {
+  const xs = runs.map(pick).filter((x) => x !== null && x !== undefined && Number.isFinite(x));
+  if (!xs.length) return '—';
+  const m = fmt(median(xs));
+  if (xs.length === 1) return m;
+  const lo = fmt(Math.min(...xs)); const hi = fmt(Math.max(...xs));
+  return lo === hi ? m : `${m} [${lo}–${hi}]`;
+}
+
+const int = (x) => Math.round(x).toLocaleString('en-US');
+const pct1 = (x) => `${x.toFixed(1)}%`;
+
+export function formatComparison(arms) {
+  const W = 36; const C = 24;
+  const out = [];
+  // The leading space is a separator, not padding: a `median [min–max]` cell can
+  // fill its column, and two of those with only padStart between them run
+  // together into one unreadable number.
+  const row = (label, cells) => out.push('  ' + label.padEnd(W) + cells.map((c) => ' ' + String(c).padStart(C - 1)).join(''));
+  const rule = (title) => out.push(`  ${title}`);
+
+  row('', arms.map((a) => a.label));
+  row('runs', arms.map((a) => a.runs.length));
+  const anyFailed = arms.some((a) => a.runs.some((r) => !r.ok));
+  if (anyFailed) row('  of which non-success', arms.map((a) => a.runs.filter((r) => !r.ok).length));
+  if (arms.some((a) => a.runs.some((r) => r.raced))) {
+    row('  MCP cold-start race', arms.map((a) => a.runs.filter((r) => r.raced).length));
+  }
+  out.push('');
+
+  rule('behavior');
+  row('  duration (s)', arms.map((a) => span(a.runs, (r) => r.dur)));
+  row('  tool calls', arms.map((a) => span(a.runs, (r) => r.tools)));
+  row('  Read', arms.map((a) => span(a.runs, (r) => r.reads)));
+  row('  Grep/Glob', arms.map((a) => span(a.runs, (r) => r.grep)));
+  row('  Bash', arms.map((a) => span(a.runs, (r) => r.bash)));
+  row('  codegraph calls', arms.map((a) => span(a.runs, (r) => r.cg)));
+  out.push('');
+
+  rule('residual context occupancy (CG-7) — tokens still resident at end of run');
+  row('  final context (tok)', arms.map((a) => span(a.runs, (r) => r.ctx, int)));
+  row('  codegraph residual (tok)', arms.map((a) => span(a.runs, (r) => r.occCg, int)));
+  row('  file-access residual (tok)', arms.map((a) => span(a.runs, (r) => r.occFile, int)));
+  row('  → retrieval residual (tok)', arms.map((a) => span(a.runs, (r) => r.occSelf, int)));
+  row('  → share of final context', arms.map((a) => span(a.runs, (r) => r.occShare, pct1)));
+  out.push('');
+
+  rule('explore sufficiency (CG-8) — pooled over every answered explore call');
+  row('  answered explore calls', arms.map((a) => a.runs.reduce((s, r) => s + r.suffAnswered, 0)));
+  for (const [key, label] of SUFFICIENCY) {
+    row(`  ${label}`, arms.map((a) => {
+      const n = a.runs.reduce((s, r) => s + r.suffCounts[key], 0);
+      const tot = a.runs.reduce((s, r) => s + r.suffAnswered, 0);
+      return tot ? `${n}  ${((n / tot) * 100).toFixed(0)}%` : '—';
+    }));
+  }
+  if (arms.some((a) => a.runs.some((r) => r.suffErrors))) {
+    row('  errored/unanswered (not bucketed)', arms.map((a) => a.runs.reduce((s, r) => s + r.suffErrors, 0)));
+  }
+  out.push('');
+
+  rule('explore allocation efficiency (CG-9) — share of returned bytes the answer cited');
+  row('  explore calls with source', arms.map((a) => a.runs.reduce((s, r) => s + r.allocCalls, 0)));
+  row('  pooled efficiency', arms.map((a) => {
+    const env = a.runs.reduce((s, r) => s + (r.allocEnvelope || 0), 0);
+    const used = a.runs.reduce((s, r) => s + (r.allocUsed || 0), 0);
+    return env ? pct1((used / env) * 100) : '—';
+  }));
+  row('  per-run efficiency', arms.map((a) =>
+    span(a.runs, (r) => (r.allocEnvelope ? (r.allocUsed / r.allocEnvelope) * 100 : null), pct1)));
+  row('  envelope (chars)', arms.map((a) => int(a.runs.reduce((s, r) => s + (r.allocEnvelope || 0), 0))));
+  out.push('');
+
+  rule('contamination — the CLI must never be how codegraph is reached');
+  row('  CLI calls that RETURNED output', arms.map((a) => a.runs.reduce((s, r) => s + r.cliContaminated, 0)));
+  row('  CLI attempts blocked', arms.map((a) => a.runs.reduce((s, r) => s + r.cliCalls, 0)));
+  const contaminated = arms.filter((a) => a.runs.some((r) => r.cliContaminated));
+  if (contaminated.length) {
+    out.push(`  !! ${contaminated.map((a) => a.label).join(', ')} reached codegraph through Bash — those runs are CONTAMINATED`);
+    out.push('     (a without-arm was not without codegraph; a with-arm has bytes attributed to Bash, not codegraph)');
+  }
+  out.push('');
+
+  out.push('  how to read this');
+  out.push('    occupancy  compare each arm\'s RETRIEVAL residual (codegraph in a with-arm,');
+  out.push('               file-access in a without-arm). Shares are Claude Code on a 200k');
+  out.push('               window and do NOT transfer to another host; the ratio does.');
+  out.push('    sufficiency  pooled across runs because it is per-CALL. "explore again" is');
+  out.push('               ambiguous by construction; "Read a file we returned" is an');
+  out.push('               allocation miss, the two recall rows are recall misses.');
+  out.push('    allocation  RELATIVE, not absolute — attribution is by citation, and an agent');
+  out.push('               can use a file without naming it. Compare builds on the SAME');
+  out.push('               question; never quote it as "codegraph wastes N% of what it returns."');
+  out.push('    all three  small-n. Runs make 1–5 explore calls, so read the range, not the');
+  out.push('               median of one run. RUNS>=2, and the 7-repo campaign for a verdict.');
+  return out.join('\n');
+}
+
+const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (isMain) {
+  const [dir, ...labels] = process.argv.slice(2);
+  if (!dir || !labels.length) {
+    console.error('usage: compare-arms.mjs <out-dir> <label> [<label> ...]');
+    process.exit(1);
+  }
+  const arms = labels.map((label) => ({ label, runs: discoverRuns(dir, label).map(measure) }));
+  const empty = arms.filter((a) => !a.runs.length);
+  if (empty.length === arms.length) {
+    console.error(`no run logs for ${labels.join('/')} in ${dir}`);
+    process.exit(1);
+  }
+  for (const a of empty) console.error(`  WARN: no run logs for arm '${a.label}' in ${dir}`);
+  console.log(`\n====== ARM COMPARISON — ${dir} ======`);
+  console.log(formatComparison(arms.filter((a) => a.runs.length)));
+}
diff --git a/scripts/agent-eval/no-cli-shim.sh b/scripts/agent-eval/no-cli-shim.sh
new file mode 100644
index 0000000..7817bb1
--- /dev/null
+++ b/scripts/agent-eval/no-cli-shim.sh
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# Keep the codegraph CLI out of an eval arm, so the MCP server is the ONLY way
+# the agent can reach codegraph. Sourced by run-all.sh and ab-new-vs-baseline.sh.
+#
+#   . "$HARNESS/no-cli-shim.sh"
+#   cg_no_cli_setup "$OUT"       # -> sets $ARM_PATH and $ARM_SETTINGS
+#   PATH="$ARM_PATH" claude … --settings "$ARM_SETTINGS"
+#
+# Why this exists, in both harnesses:
+#
+#   with/without (run-all.sh)  The without-arm gets an empty MCP config but still
+#   has Bash, and the target repo carries the .codegraph/ index. Agents find that:
+#   14 of 15 without-arm runs in one 7-repo pass ran `codegraph explore` through
+#   Bash (one via `ls .codegraph && codegraph explore …`), so that arm was
+#   measuring codegraph-over-CLI, not codegraph-absent.
+#
+#   new/baseline (ab-new-vs-baseline.sh)  Both arms are codegraph-on, so a CLI
+#   call is not a with/without leak — it is an ATTRIBUTION leak, and it breaks all
+#   three feedback metrics at once. Output that arrives through Bash is charged to
+#   Bash (understating occupancy), and an explore issued through the CLI is not a
+#   tool call at all, so it never reaches the sufficiency classifier or the
+#   allocation parse. A run that shells out silently drops calls from the numbers.
+#
+# Two layers, because one was not enough:
+#
+#   1. PATH. The binary usually shares a directory with tools the run needs
+#      (claude itself lives next to it here), so dropping the whole directory is
+#      not an option. Substitute an equivalent directory IN PLACE: symlinks to
+#      every entry except codegraph, keeping PATH order and precedence intact.
+#   2. A PreToolUse hook. An agent denied `codegraph` ran
+#      `find / -maxdepth 4 -iname "*codegraph*"`, found the binary, and invoked it
+#      by ABSOLUTE PATH — so block the invocation itself. Written into the output
+#      dir as a run artifact rather than a repo file, same as the MCP configs.
+#
+# Neither layer is a substitute for the counter: parse-run.mjs flags any Bash
+# command that named codegraph, separating attempts it blocked (no output entered
+# the window) from calls that RETURNED output. Prevention fails silently the next
+# time the binary lands somewhere new; the counter does not.
+
+# Command positions only: `grep codegraph x`, `ls .codegraph` and
+# `which codegraph` are looking, not using, and pass through.
+CG_CMD_RE='(^|[;&|(]|&&|\|\||\$\(|`)[[:space:]]*([A-Za-z_][A-Za-z0-9_]*=[^[:space:]]*[[:space:]]+)*[A-Za-z0-9_./~-]*codegraph([[:space:]]|$)'
+
+cg_no_cli_setup() {
+  local out="${1:?cg_no_cli_setup <out-dir>}"
+  local shim="$out/nocg-bin"
+
+  rm -rf "$shim"; mkdir -p "$shim"
+  local built="" d e
+  local IFS=:
+  for d in $PATH; do
+    [ -n "$d" ] || continue
+    if [ -x "$d/codegraph" ]; then
+      for e in "$d"/*; do
+        [ "$(basename "$e")" = codegraph ] && continue
+        ln -sf "$e" "$shim/" 2>/dev/null
+      done
+      d="$shim"
+    fi
+    built="${built:+$built:}$d"
+  done
+  unset IFS
+  ARM_PATH="$built"
+
+  if PATH="$ARM_PATH" command -v codegraph >/dev/null 2>&1; then
+    echo "WARNING: 'codegraph' is still on the arm PATH — runs will be contaminated"
+  fi
+  for e in claude node; do
+    PATH="$ARM_PATH" command -v "$e" >/dev/null || { echo "sanitized PATH lost '$e' — refusing to run"; return 1; }
+  done
+
+  command -v jq >/dev/null || { echo "jq is required for the CLI-block hook — install it or the arms will be contaminated"; return 1; }
+  cat > "$out/no-cli-hook.sh" <<HOOK
+#!/usr/bin/env bash
+# Deny Bash invocations of the codegraph CLI so the MCP server stays the A/B's
+# single variable. Looking for it is fine; running it is not.
+set -uo pipefail
+cmd="\$(cat | jq -r '.tool_input.command // empty' 2>/dev/null)"
+if printf '%s' "\$cmd" | grep -Eq '$CG_CMD_RE'; then
+  msg="The codegraph CLI is not available in this session. Answer using the tools you have."
+  jq -n --arg m "\$msg" '{reason:\$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:\$m}}'
+fi
+exit 0
+HOOK
+  chmod +x "$out/no-cli-hook.sh"
+  cat > "$out/hook-settings.json" <<JSON
+{"hooks":{"PreToolUse":[{"matcher":"Bash","hooks":[{"type":"command","command":"bash $out/no-cli-hook.sh"}]}]}}
+JSON
+  ARM_SETTINGS="$out/hook-settings.json"
+
+  # Prove the hook denies a real invocation and lets a mere mention through.
+  cg_no_cli_probe() { printf '{"tool_input":{"command":%s}}' "$2" | bash "$1/no-cli-hook.sh" | grep -c deny; }
+  [ "$(cg_no_cli_probe "$out" '"/Users/x/.local/bin/codegraph explore \"q\""')" = 1 ] || { echo "hook fails to block an absolute-path invocation"; return 1; }
+  [ "$(cg_no_cli_probe "$out" '"grep -rn codegraph src/"')" = 0 ] || { echo "hook over-blocks a plain mention"; return 1; }
+  return 0
+}
diff --git a/scripts/agent-eval/parse-bench-readme.mjs b/scripts/agent-eval/parse-bench-readme.mjs
index 5766005..b836d04 100644
--- a/scripts/agent-eval/parse-bench-readme.mjs
+++ b/scripts/agent-eval/parse-bench-readme.mjs
@@ -17,7 +17,7 @@
 // Usage: node parse-bench-readme.mjs [/tmp/ab-readme]
 import { existsSync, readdirSync } from 'fs';
 import { join } from 'path';
-import { parseSession } from './parse-run.mjs';
+import { parseSession, SUFFICIENCY } from './parse-run.mjs';
 
 const ROOT = process.argv[2] || '/tmp/ab-readme';
 const REPOS = ['vscode', 'excalidraw', 'django', 'tokio', 'okhttp', 'gin', 'alamofire'];
@@ -55,6 +55,16 @@ function parse(dir, label) {
     occShareCtx: o.ctxFinal > 0 ? ((o.residual.codegraph + o.residualFileAccess) / o.ctxFinal) * 100 : 0,
     occShareWin: ((o.residual.codegraph + o.residualFileAccess) / o.windowTokens) * 100,
     window: o.windowTokens,
+    // The other two feedback metrics, carried per run so the campaign can pool
+    // them. Both are with-arm-only in practice — a without-arm makes no explore
+    // calls, so it has nothing to be sufficient about and no bytes to allocate.
+    suffAnswered: s.sufficiency.answered,
+    suffCounts: s.sufficiency.counts,
+    // Byte-weighted, so a run with no explore contributes NOTHING rather than a
+    // zero; a zero would drag a repo toward "wasteful" for never spending a byte.
+    allocUsed: s.allocation.envelope ? s.allocation.used : 0,
+    allocEnvelope: s.allocation.envelope,
+    allocCalls: s.allocation.calls.length,
   };
 }
 
@@ -161,3 +171,50 @@ if (!anyMulti) {
     `follow-ups (see run-all.sh) to measure the regime this metric is actually about.`
   );
 }
+
+// ---- Table 3: sufficiency + allocation, WITH arm only. ---------------------
+// Occupancy says what a response COST; these two say whether it was enough and
+// whether it spent its bytes on the right files. A campaign that reports only
+// occupancy cannot tell a tighter response from a worse one.
+//
+// Pooled per repo, not median-of-runs: both are per-CALL quantities (sufficiency
+// counts explores, allocation weights by bytes), and a repo contributes 2-15
+// calls across its runs. Median-of-run-percentages would weight a 1-call run the
+// same as a 5-call one.
+console.log(`\n\nEXPLORE SUFFICIENCY + ALLOCATION EFFICIENCY — with-arm only, pooled over runs`);
+console.log(`(sufficiency = what the agent did NEXT · allocation = share of returned bytes the answer cited)\n`);
+console.log('repo        calls   again  read-ret  read-miss  grep   MOVED ON     alloc eff   envelope');
+const totals = { answered: 0, counts: Object.fromEntries(SUFFICIENCY.map(([k]) => [k, 0])), used: 0, env: 0, calls: 0 };
+for (const { repo, W } of rows) {
+  if (!W.length) { console.log(`${repo.padEnd(11)} (no with-arm runs)`); continue; }
+  const answered = W.reduce((s, r) => s + r.suffAnswered, 0);
+  const cnt = (k) => W.reduce((s, r) => s + r.suffCounts[k], 0);
+  const env = W.reduce((s, r) => s + r.allocEnvelope, 0);
+  const used = W.reduce((s, r) => s + r.allocUsed, 0);
+  totals.answered += answered; totals.used += used; totals.env += env;
+  totals.calls += W.reduce((s, r) => s + r.allocCalls, 0);
+  for (const [k] of SUFFICIENCY) totals.counts[k] += cnt(k);
+  const cell = (k) => (answered ? `${cnt(k)} ${Math.round((cnt(k) / answered) * 100)}%` : '—').padEnd(9);
+  console.log(
+    `${repo.padEnd(11)} ${String(answered).padEnd(7)} ` +
+    `${cell('explore_again')}${cell('read_returned')}${cell('read_missed')}${cell('search')}` +
+    `${(answered ? `${cnt('sufficient')} ${Math.round((cnt('sufficient') / answered) * 100)}%` : '—').padEnd(13)}` +
+    `${(env ? `${((used / env) * 100).toFixed(1)}%` : '—').padEnd(12)}${fmtTok(env)}`
+  );
+}
+const tp = (k) => totals.answered ? `${totals.counts[k]} (${Math.round((totals.counts[k] / totals.answered) * 100)}%)` : '—';
+console.log(
+  `\nPOOLED (${totals.answered} answered explore calls): ` +
+  SUFFICIENCY.map(([k, label]) => `${label} ${tp(k)}`).join(' · ')
+);
+console.log(
+  `POOLED allocation efficiency: ${totals.env ? ((totals.used / totals.env) * 100).toFixed(1) + '%' : '—'} ` +
+  `over ${totals.calls} calls / ${fmtTok(totals.env)} chars`
+);
+console.log(
+  `\nHOW TO READ: "read-ret" (Read a file we RETURNED) is an allocation miss — right file,\n` +
+  `wrong bytes; "read-miss" and "grep" are recall misses. "again" is ambiguous by construction.\n` +
+  `Allocation efficiency is RELATIVE — attribution is by citation, so it compares BUILDS on the\n` +
+  `same questions and is not a claim that codegraph wasted the remainder. Full guidance:\n` +
+  `docs/benchmarks/agent-eval-feedback-metrics.md`
+);
diff --git a/scripts/agent-eval/parse-run.mjs b/scripts/agent-eval/parse-run.mjs
index 2e04501..58bff56 100644
--- a/scripts/agent-eval/parse-run.mjs
+++ b/scripts/agent-eval/parse-run.mjs
@@ -3,12 +3,16 @@
 // RESIDUAL CONTEXT OCCUPANCY — how many tokens of the context window each tool
 // family's responses still occupy when the run ends.
 //
-// Usage: parse-run.mjs <run.jsonl> [run.t2.jsonl ...] [--envelope] [--answer <glob>]...
+// Usage: parse-run.mjs <run.jsonl> [run.t2.jsonl ...] [--brief] [--envelope] [--answer <glob>]...
 //   Multiple files = one multi-turn session's segments, IN ORDER (run-all.sh
 //   writes run-<label>.jsonl, run-<label>.t2.jsonl, … for a `Q1||Q2||Q3` set).
 //   `--resume` does not replay prior messages, so the segments concatenate
 //   cleanly and token accounting carries across the boundary.
 //
+//   `--brief` drops the numbered tool-call transcript and keeps everything else,
+//   for harnesses that print one of these blocks per run (ab-new-vs-baseline.sh
+//   at RUNS>=2 is otherwise mostly call listings).
+//
 //   Every run also reports EXPLORE SUFFICIENCY — each codegraph_explore call
 //   bucketed by what the agent did next (see classifySufficiency) — and EXPLORE
 //   ALLOCATION EFFICIENCY, the share of the bytes explore returned that belonged
@@ -729,7 +733,7 @@ const TRANSPARENT_TOOLS = new Set(['ToolSearch', 'TodoWrite']);
 const DELEGATION_TOOLS = new Set(['Agent', 'Task']);
 
 /** Buckets, worst → best. Labels double as the summary rows. */
-const SUFFICIENCY = [
+export const SUFFICIENCY = [
   ['explore_again', 'explore again', 'insufficient: did not answer'],
   ['read_returned', 'Read a file we returned', 'allocation: right file, wrong bytes'],
   ['read_missed', 'Read a file we did not return', 'recall: file never surfaced'],
@@ -1340,12 +1344,14 @@ if (isMain) {
   const files = [];
   const answerGlobs = [];
   let wantEnvelope = false;
+  let brief = false;
   for (let i = 0; i < argv.length; i++) {
     if (argv[i] === '--envelope') wantEnvelope = true;
+    else if (argv[i] === '--brief') brief = true;
     else if (argv[i] === '--answer') { answerGlobs.push(argv[++i]); wantEnvelope = true; }
     else if (!argv[i].startsWith('--')) files.push(argv[i]);
   }
-  if (!files.length) { console.error('usage: parse-run.mjs <run.jsonl> [run.t2.jsonl ...] [--envelope] [--answer <glob>]...  |  --selftest'); process.exit(1); }
+  if (!files.length) { console.error('usage: parse-run.mjs <run.jsonl> [run.t2.jsonl ...] [--brief] [--envelope] [--answer <glob>]...  |  --selftest'); process.exit(1); }
   const s = parseSession(files);
 
   console.log(`\n=== ${files.map((f) => f.split('/').pop()).join(' + ')} ===`);
@@ -1354,7 +1360,7 @@ if (isMain) {
   else if (s.cliCalls) console.log(`   (${s.cliCalls} codegraph CLI attempt${s.cliCalls === 1 ? '' : 's'} blocked — no output entered the window)`);
   console.log(`\nTool calls (${s.toolCalls.length}):`);
   console.log('  by type:', JSON.stringify(s.counts));
-  s.toolCalls.forEach((tc, i) => console.log(`  ${i + 1}. ${tc}`));
+  if (!brief) s.toolCalls.forEach((tc, i) => console.log(`  ${i + 1}. ${tc}`));
 
   if (s.result) {
     const seg = s.results.length > 1 ? ` | ${s.results.length} segments (${s.results.map((r) => r.subtype).join(',')})` : '';
diff --git a/scripts/agent-eval/run-all.sh b/scripts/agent-eval/run-all.sh
index 28ccb6f..7b7d985 100755
--- a/scripts/agent-eval/run-all.sh
+++ b/scripts/agent-eval/run-all.sh
@@ -6,6 +6,15 @@
 #
 # Usage: run-all.sh <repo-path> "<question>" [headless|tmux|all]
 #
+# Each headless arm reports the three feedback metrics (parse-run.mjs prints all
+# three under every run, and compare-arms.mjs puts the arms side by side at the
+# end when both ran):
+#   residual context occupancy (CG-7)  window still held by the arm's retrieval
+#   explore sufficiency        (CG-8)  was the response ENOUGH — the agent's next act
+#   allocation efficiency      (CG-9)  share of returned bytes the answer cited
+# docs/benchmarks/agent-eval-feedback-metrics.md is the entry point; the three
+# per-metric docs it links carry the caveats.
+#
 # MULTI-TURN: separate questions with "||" to run them as ONE session —
 #   run-all.sh <repo> "How does X work?||Where is Y handled in that path?"
 # Turn 1 runs normally; every later turn `--resume`s the same session, so the
@@ -45,72 +54,12 @@ export CODEGRAPH_NO_PROMPT_HOOK=1
 
 # Hide the codegraph CLI from BOTH arms, so the only way to reach codegraph is
 # the MCP server wired below — which is what makes it the A/B's single variable.
-#
-# Both arms have Bash, and the target repo carries the .codegraph/ index the
-# with-arm needs. Agents FIND that: 14 of 15 without-arm runs in one 7-repo pass
-# ran `codegraph explore` through Bash (one via `ls .codegraph && codegraph
-# explore …`), so that arm was measuring codegraph-over-CLI, not
-# codegraph-absent. It matters in the with-arm too — output that arrives through
-# Bash is attributed to Bash, understating what codegraph itself occupies.
-#
-# The binary usually shares a directory with tools the run needs (claude itself
-# lives next to it here), so dropping the whole directory is not an option.
-# Substitute an equivalent directory IN PLACE: symlinks to every entry except
-# codegraph, keeping PATH order and precedence intact.
-SHIM_BIN="$OUT/nocg-bin"
-rm -rf "$SHIM_BIN"; mkdir -p "$SHIM_BIN"
-sanitized_path() {
-  local out="" d e
-  local IFS=:
-  for d in $PATH; do
-    [ -n "$d" ] || continue
-    if [ -x "$d/codegraph" ]; then
-      for e in "$d"/*; do
-        [ "$(basename "$e")" = codegraph ] && continue
-        ln -sf "$e" "$SHIM_BIN/" 2>/dev/null
-      done
-      d="$SHIM_BIN"
-    fi
-    out="${out:+$out:}$d"
-  done
-  printf '%s' "$out"
-}
-ARM_PATH="$(sanitized_path)"
-if PATH="$ARM_PATH" command -v codegraph >/dev/null 2>&1; then
-  echo "WARNING: 'codegraph' is still on the arm PATH — runs will be contaminated"
-fi
-for t in claude node; do
-  PATH="$ARM_PATH" command -v "$t" >/dev/null || { echo "sanitized PATH lost '$t' — refusing to run"; exit 1; }
-done
-
-# Hiding it from PATH is not enough. An agent denied `codegraph` ran
-# `find / -maxdepth 4 -iname "*codegraph*"`, found the binary, and invoked it by
-# ABSOLUTE PATH — so block the invocation itself with a PreToolUse hook. Written
-# into $OUT as a run artifact rather than a repo file, same as the MCP configs.
-# The pattern deliberately matches only COMMAND positions: `grep codegraph x`,
-# `ls .codegraph` and `which codegraph` are looking, not using, and pass through.
-CG_CMD_RE='(^|[;&|(]|&&|\|\||\$\(|`)[[:space:]]*([A-Za-z_][A-Za-z0-9_]*=[^[:space:]]*[[:space:]]+)*[A-Za-z0-9_./~-]*codegraph([[:space:]]|$)'
-cat > "$OUT/no-cli-hook.sh" <<HOOK
-#!/usr/bin/env bash
-# Deny Bash invocations of the codegraph CLI so the MCP server stays the A/B's
-# single variable. Looking for it is fine; running it is not.
-set -uo pipefail
-cmd="\$(cat | jq -r '.tool_input.command // empty' 2>/dev/null)"
-if printf '%s' "\$cmd" | grep -Eq '$CG_CMD_RE'; then
-  msg="The codegraph CLI is not available in this session. Answer using the tools you have."
-  jq -n --arg m "\$msg" '{reason:\$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:\$m}}'
-fi
-exit 0
-HOOK
-chmod +x "$OUT/no-cli-hook.sh"
-cat > "$OUT/hook-settings.json" <<JSON
-{"hooks":{"PreToolUse":[{"matcher":"Bash","hooks":[{"type":"command","command":"bash $OUT/no-cli-hook.sh"}]}]}}
-JSON
-command -v jq >/dev/null || { echo "jq is required for the CLI-block hook — install it or the arms will be contaminated"; exit 1; }
-# Prove the hook denies a real invocation and lets a mere mention through.
-_probe() { printf '{"tool_input":{"command":%s}}' "$1" | bash "$OUT/no-cli-hook.sh" | grep -c deny; }
-[ "$(_probe '"/Users/x/.local/bin/codegraph explore \"q\""')" = 1 ] || { echo "hook fails to block an absolute-path invocation"; exit 1; }
-[ "$(_probe '"grep -rn codegraph src/"')" = 0 ] || { echo "hook over-blocks a plain mention"; exit 1; }
+# Two layers (sanitized PATH + a PreToolUse hook that blocks absolute-path
+# invocations), both in no-cli-shim.sh, which ab-new-vs-baseline.sh shares; the
+# reasoning and the incident that forced each layer are documented there.
+# Sets $ARM_PATH and $ARM_SETTINGS, and aborts if either layer fails its probe.
+. "$HARNESS/no-cli-shim.sh"
+cg_no_cli_setup "$OUT" || exit 1
 
 [ -n "$CG_BIN" ] || { echo "no codegraph binary on PATH (set CG_BIN)"; exit 1; }
 [ -d "$REPO/.codegraph" ] || { echo "no .codegraph index at $REPO — index it first"; exit 1; }
@@ -158,7 +107,7 @@ headless() {
         --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
         --max-budget-usd 4 \
         --strict-mcp-config --mcp-config "$cfg" \
-        --settings "$OUT/hook-settings.json" \
+        --settings "$ARM_SETTINGS" \
         ${resume[@]+"${resume[@]}"} \
         </dev/null > "$out" 2>>"$OUT/run-$label.err" )
     echo "exit $? -> $out ($(wc -l < "$out" | tr -d ' ') lines) [turn $seg/${#TURNS[@]}]"
@@ -179,6 +128,12 @@ ARMS="${CG_ARMS:-both}"
 if [ "$MODE" = headless ] || [ "$MODE" = all ]; then
   case "$ARMS" in both|with)    headless "headless-with"    "$OUT/mcp-codegraph.json";; esac
   case "$ARMS" in both|without) headless "headless-without" "$OUT/mcp-empty.json";; esac
+  # Both arms' three metrics on one screen. The per-arm blocks above say WHY a
+  # number moved (which query fell short, which file was never cited); this says
+  # whether it moved at all. CG_ARMS=with|without leaves one arm's logs from an
+  # earlier invocation in $OUT, and comparing against those is the point of the
+  # split — so this runs whichever arms have logs, not only a fresh pair.
+  node "$HARNESS/compare-arms.mjs" "$OUT" headless-with headless-without 2>&1 || true
 fi
 
 if [ "$MODE" = tmux ] || [ "$MODE" = all ]; then