From e35d4861e02b1e0b0efd1088dfc1bf8c6eddf069 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 16:10:32 -0500 Subject: [PATCH] =?UTF-8?q?test(agent-eval):=20block=20the=20codegraph=20C?= =?UTF-8?q?LI=20outright=20=E2=80=94=20hiding=20it=20from=20PATH=20was=20n?= =?UTF-8?q?ot=20enough=20(CG-7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An agent denied `codegraph` on PATH ran `find / -maxdepth 4 -iname "*codegraph*"`, found the binary, and invoked it by ABSOLUTE PATH — 12 times in one without-arm run. So block the invocation itself with a PreToolUse hook on Bash, written into the run's output dir as an artifact alongside the MCP configs rather than as a repo file. The pattern matches command positions only, so looking is still allowed and only using is denied: `grep codegraph src/`, `ls .codegraph` and `which codegraph` pass through, while `codegraph explore`, `/abs/path/codegraph …`, `cd x && codegraph …` and `VAR=1 codegraph …` are refused. run-all.sh proves both directions at startup and refuses to run if either fails. parse-run.mjs's detector uses the same rule, so prevention and detection cannot drift — and it no longer false-positives on the corpus path, which contains the word codegraph. Verified end-to-end: the without-arm now probes with `ls .codegraph; which codegraph`, finds nothing usable, and falls back to Read/Bash. --- docs/benchmarks/residual-context-occupancy.md | 35 +++++++++++++++++++ scripts/agent-eval/parse-run.mjs | 7 +++- scripts/agent-eval/run-all.sh | 30 ++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/docs/benchmarks/residual-context-occupancy.md b/docs/benchmarks/residual-context-occupancy.md index bb88f90..042ab6a 100644 --- a/docs/benchmarks/residual-context-occupancy.md +++ b/docs/benchmarks/residual-context-occupancy.md @@ -126,6 +126,41 @@ question. --- +## The without-arm was never actually without codegraph + +Establishing this baseline turned up a contamination channel that had been open +the whole time, and it invalidates any number this harness produced for an arm +that had Bash. + +The without-arm gets an empty MCP config, so it has no codegraph tool. It still +has **Bash** — and the target repo still carries the `.codegraph/` index the +with-arm needs, with the `codegraph` binary on PATH. Agents find that. In the +first clean-looking 7-repo pass, **14 of 15 without-arm runs ran `codegraph +explore` through Bash**, one of them by way of `ls .codegraph && codegraph +explore …`. That arm was measuring codegraph-over-CLI against codegraph-over-MCP, +not codegraph against its absence. + +It cuts the other way too. When the *with*-arm shells out, the output arrives as +a Bash result and is attributed to Bash — understating what codegraph itself +occupies. One of 15 with-arm runs did this. + +The fix is in `run-all.sh`: both arms now run on a PATH where the CLI is hidden, +so the MCP server is the only way to reach codegraph and stays the A/B's single +variable. The binary usually shares a directory with tools the run needs — here +`claude` sits right next to it — so the directory is substituted in place by one +of symlinks to every entry except `codegraph`, which keeps PATH order and +precedence intact. The run aborts if `claude` or `node` did not survive. + +Prevention alone would fail silently the next time the binary lands somewhere +new, so there is detection as well: `parse-run.mjs` flags any Bash command naming +codegraph, and `parse-bench-readme.mjs` drops contaminated without-arm runs from +the aggregate (`CG_INCLUDE_CONTAMINATED=1` keeps them). + +**Anyone re-reading older A/B results from this harness should assume the +without-arm may have been using codegraph.** + +--- + ## Baseline: the 7 README repos diff --git a/scripts/agent-eval/parse-run.mjs b/scripts/agent-eval/parse-run.mjs index ea69b76..348617e 100644 --- a/scripts/agent-eval/parse-run.mjs +++ b/scripts/agent-eval/parse-run.mjs @@ -59,6 +59,11 @@ const FAMILIES = ['codegraph', 'read', 'search', 'bash', 'other']; // The without-arm's way of getting the same bytes: reading and searching files. const FILE_ACCESS = ['read', 'search', 'bash']; +// A Bash command that INVOKES the codegraph CLI, in any command position and by +// any path. Mentions are not invocations: `grep codegraph src/`, `ls .codegraph` +// and `which codegraph` all pass. Kept in step with run-all.sh's blocking hook. +const CG_CLI_RE = /(^|[;&|(]|&&|\|\||\$\(|`)\s*(?:[A-Za-z_]\w*=\S*\s+)*[\w./~-]*codegraph(\s|$)/; + const textOf = (content) => Array.isArray(content) ? content.map((c) => c.text ?? (typeof c === 'string' ? c : JSON.stringify(c))).join('') : typeof content === 'string' ? content @@ -131,7 +136,7 @@ export function parseSession(files) { // An arm with no codegraph MCP can still shell out to the CLI — the // target repo carries the .codegraph/ index and the binary is on // PATH. That silently turns a "without" arm into codegraph-over-CLI. - if (/\bcodegraph\b/.test(b.input?.command ?? '')) cliCalls++; + if (CG_CLI_RE.test(b.input?.command ?? '')) cliCalls++; } else if (b.name === 'Read') detail = ` ${(b.input?.file_path ?? '').split('/').slice(-1)[0]}`; toolCalls.push(`${b.name}${detail}`); diff --git a/scripts/agent-eval/run-all.sh b/scripts/agent-eval/run-all.sh index 349818b..28ccb6f 100755 --- a/scripts/agent-eval/run-all.sh +++ b/scripts/agent-eval/run-all.sh @@ -83,6 +83,35 @@ for t in claude node; do PATH="$ARM_PATH" command -v "$t" >/dev/null || { echo "sanitized PATH lost '$t' — refusing to run"; exit 1; } done +# Hiding it from PATH is not enough. An agent denied `codegraph` ran +# `find / -maxdepth 4 -iname "*codegraph*"`, found the binary, and invoked it by +# ABSOLUTE PATH — so block the invocation itself with a PreToolUse hook. Written +# into $OUT as a run artifact rather than a repo file, same as the MCP configs. +# The pattern deliberately matches only COMMAND positions: `grep codegraph x`, +# `ls .codegraph` and `which codegraph` are looking, not using, and pass through. +CG_CMD_RE='(^|[;&|(]|&&|\|\||\$\(|`)[[:space:]]*([A-Za-z_][A-Za-z0-9_]*=[^[:space:]]*[[:space:]]+)*[A-Za-z0-9_./~-]*codegraph([[:space:]]|$)' +cat > "$OUT/no-cli-hook.sh" < "$OUT/hook-settings.json" </dev/null || { echo "jq is required for the CLI-block hook — install it or the arms will be contaminated"; exit 1; } +# Prove the hook denies a real invocation and lets a mere mention through. +_probe() { printf '{"tool_input":{"command":%s}}' "$1" | bash "$OUT/no-cli-hook.sh" | grep -c deny; } +[ "$(_probe '"/Users/x/.local/bin/codegraph explore \"q\""')" = 1 ] || { echo "hook fails to block an absolute-path invocation"; exit 1; } +[ "$(_probe '"grep -rn codegraph src/"')" = 0 ] || { echo "hook over-blocks a plain mention"; exit 1; } + [ -n "$CG_BIN" ] || { echo "no codegraph binary on PATH (set CG_BIN)"; exit 1; } [ -d "$REPO/.codegraph" ] || { echo "no .codegraph index at $REPO — index it first"; exit 1; } case "$MODE" in headless|tmux|all) ;; *) echo "mode must be headless|tmux|all (got '$MODE')"; exit 1;; esac @@ -129,6 +158,7 @@ headless() { --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \ --max-budget-usd 4 \ --strict-mcp-config --mcp-config "$cfg" \ + --settings "$OUT/hook-settings.json" \ ${resume[@]+"${resume[@]}"} \ "$out" 2>>"$OUT/run-$label.err" ) echo "exit $? -> $out ($(wc -l < "$out" | tr -d ' ') lines) [turn $seg/${#TURNS[@]}]"