test(agent-eval): block the codegraph CLI outright — hiding it from PATH was not enough (CG-7)
An agent denied `codegraph` on PATH ran `find / -maxdepth 4 -iname "*codegraph*"`, found the binary, and invoked it by ABSOLUTE PATH — 12 times in one without-arm run. So block the invocation itself with a PreToolUse hook on Bash, written into the run's output dir as an artifact alongside the MCP configs rather than as a repo file. The pattern matches command positions only, so looking is still allowed and only using is denied: `grep codegraph src/`, `ls .codegraph` and `which codegraph` pass through, while `codegraph explore`, `/abs/path/codegraph …`, `cd x && codegraph …` and `VAR=1 codegraph …` are refused. run-all.sh proves both directions at startup and refuses to run if either fails. parse-run.mjs's detector uses the same rule, so prevention and detection cannot drift — and it no longer false-positives on the corpus path, which contains the word codegraph. Verified end-to-end: the without-arm now probes with `ls .codegraph; which codegraph`, finds nothing usable, and falls back to Read/Bash.
This commit is contained in:
@@ -59,6 +59,11 @@ const FAMILIES = ['codegraph', 'read', 'search', 'bash', 'other'];
|
||||
// The without-arm's way of getting the same bytes: reading and searching files.
|
||||
const FILE_ACCESS = ['read', 'search', 'bash'];
|
||||
|
||||
// A Bash command that INVOKES the codegraph CLI, in any command position and by
|
||||
// any path. Mentions are not invocations: `grep codegraph src/`, `ls .codegraph`
|
||||
// and `which codegraph` all pass. Kept in step with run-all.sh's blocking hook.
|
||||
const CG_CLI_RE = /(^|[;&|(]|&&|\|\||\$\(|`)\s*(?:[A-Za-z_]\w*=\S*\s+)*[\w./~-]*codegraph(\s|$)/;
|
||||
|
||||
const textOf = (content) =>
|
||||
Array.isArray(content) ? content.map((c) => c.text ?? (typeof c === 'string' ? c : JSON.stringify(c))).join('')
|
||||
: typeof content === 'string' ? content
|
||||
@@ -131,7 +136,7 @@ export function parseSession(files) {
|
||||
// An arm with no codegraph MCP can still shell out to the CLI — the
|
||||
// target repo carries the .codegraph/ index and the binary is on
|
||||
// PATH. That silently turns a "without" arm into codegraph-over-CLI.
|
||||
if (/\bcodegraph\b/.test(b.input?.command ?? '')) cliCalls++;
|
||||
if (CG_CLI_RE.test(b.input?.command ?? '')) cliCalls++;
|
||||
}
|
||||
else if (b.name === 'Read') detail = ` ${(b.input?.file_path ?? '').split('/').slice(-1)[0]}`;
|
||||
toolCalls.push(`${b.name}${detail}`);
|
||||
|
||||
Reference in New Issue
Block a user