Merge pull request #1527 from colbymchenry/bugfix/CG-38
CG-38: guarantee an agent-named symbol renders, wherever it sits
This commit is contained in:
Colby Mchenry
Executable
+163
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* "Did the symbol the agent NAMED actually render?" (CG-38).
|
||||
*
|
||||
* This is the measurement the whole CG-24 epic was missing. Every other probe
|
||||
* here scores the response in AGGREGATE — `probe-suite-envelope.mjs` measures how
|
||||
* much source came back, `probe-file-spend.mjs` measures whether the bytes went
|
||||
* to the files that earned them, `probe-allocation.mjs` measures group shares.
|
||||
* All three are green on a response that returns 25K of source from the right
|
||||
* file and still omits the one function the agent asked for by name. That is
|
||||
* exactly what CG-38 was: `queueMessage` at L1087 of a 1,414-line file, whose
|
||||
* file won rank #1 with 67% of the envelope, never rendered — the agent got a
|
||||
* same-stem `QueuedMessage` INTERFACE at L70 instead and had to Read the file.
|
||||
*
|
||||
* So the assertion here is per-SYMBOL and binary: for each named symbol, does its
|
||||
* definition line appear in the rendered source? Nothing else can substitute —
|
||||
* not the file being present, not its share, not its byte count.
|
||||
*
|
||||
* Usage (needs a current `npm run build`):
|
||||
* node scripts/agent-eval/probe-named-symbol.mjs
|
||||
* node scripts/agent-eval/probe-named-symbol.mjs --verbose
|
||||
* # any indexed repo, ad hoc:
|
||||
* node scripts/agent-eval/probe-named-symbol.mjs <repo> "<query>" sym1 sym2
|
||||
*
|
||||
* Exit code is 1 when any expected symbol is missing, so this can gate.
|
||||
*/
|
||||
import { cpSync, mkdtempSync, rmSync, readFileSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve, dirname } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO = resolve(HERE, '..', '..');
|
||||
|
||||
const load = async (rel) => import(pathToFileURL(resolve(REPO, rel)).href);
|
||||
const idxMod = await load('dist/index.js');
|
||||
const toolsMod = await load('dist/mcp/tools.js');
|
||||
const CodeGraph = idxMod.default?.default ?? idxMod.default ?? idxMod.CodeGraph;
|
||||
const { ToolHandler } = toolsMod;
|
||||
|
||||
/**
|
||||
* The fixture cases. `symbols` are what the agent names; each must come back with
|
||||
* its DEFINITION rendered. The queries deliberately cover both shapes the bug was
|
||||
* reported on — a bare symbol bag and a prose question — because the failure had
|
||||
* a different cause on each and a fix for one does not imply the other.
|
||||
*/
|
||||
const FIXTURE = '__tests__/fixtures/tail-render-ts';
|
||||
const CASES = [
|
||||
{
|
||||
id: 'tail-symbol-bag',
|
||||
why: 'two sibling closures past L1000, named directly; neither calls the other',
|
||||
query: 'queueMessage flushQueuedMessages',
|
||||
symbols: ['queueMessage', 'flushQueuedMessages'],
|
||||
},
|
||||
{
|
||||
id: 'tail-prose',
|
||||
why: 'same two symbols named inside a prose question',
|
||||
query: 'how does queueMessage hand its entries to flushQueuedMessages',
|
||||
symbols: ['queueMessage', 'flushQueuedMessages'],
|
||||
},
|
||||
{
|
||||
id: 'tail-with-decoy',
|
||||
why: 'the same-stem QueuedMessage interface at L70 must not stand in for the functions',
|
||||
query: 'explain queueMessage, removeQueuedMessage and flushQueuedMessages',
|
||||
symbols: ['queueMessage', 'removeQueuedMessage', 'flushQueuedMessages'],
|
||||
},
|
||||
];
|
||||
|
||||
/** Every `<n>\t<text>` line number present in the response's source blocks. */
|
||||
function renderedLines(response) {
|
||||
const out = new Set();
|
||||
for (const m of response.matchAll(/^(\d+)\t/gm)) out.add(Number(m[1]));
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* A symbol counts as rendered only when its DECLARATION line is among the lines
|
||||
* the response actually sent — not when its name merely appears somewhere (it
|
||||
* shows up in the section header symbol list and in call sites regardless, which
|
||||
* is precisely how this defect hid for a whole epic).
|
||||
*/
|
||||
function check(cg, response, names) {
|
||||
const lines = renderedLines(response);
|
||||
return names.map((name) => {
|
||||
const node = (cg.getNodesByName?.(name) ?? []).find((n) => n.startLine > 0);
|
||||
return {
|
||||
name,
|
||||
file: node?.filePath ?? '(not indexed)',
|
||||
line: node?.startLine ?? 0,
|
||||
rendered: !!node && lines.has(node.startLine),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function runCase(root, { query, symbols }) {
|
||||
const cg = CodeGraph.openSync(root);
|
||||
try {
|
||||
const res = await new ToolHandler(cg).execute('codegraph_explore', { query });
|
||||
const response = res.content?.[0]?.text ?? '';
|
||||
return { response, results: check(cg, response, symbols) };
|
||||
} finally {
|
||||
try { cg.close?.(); } catch { /* already closed */ }
|
||||
}
|
||||
}
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const verbose = argv.includes('--verbose');
|
||||
const positional = argv.filter((a) => !a.startsWith('--'));
|
||||
|
||||
let failures = 0;
|
||||
let checked = 0;
|
||||
|
||||
if (positional.length >= 3) {
|
||||
// Ad-hoc mode: <repo> "<query>" sym...
|
||||
const [repo, query, ...symbols] = positional;
|
||||
const { response, results } = await runCase(resolve(repo), { query, symbols });
|
||||
console.log(`\n${repo}\n query "${query}" · ${response.length} chars\n`);
|
||||
for (const r of results) {
|
||||
checked += 1;
|
||||
if (!r.rendered) failures += 1;
|
||||
console.log(` ${r.rendered ? 'PASS' : 'FAIL'} ${r.name} ${r.file}:${r.line}`);
|
||||
}
|
||||
} else {
|
||||
const src = join(REPO, FIXTURE);
|
||||
if (!existsSync(src)) {
|
||||
console.error(`fixture missing: ${src}`);
|
||||
process.exit(2);
|
||||
}
|
||||
const dir = mkdtempSync(join(tmpdir(), 'cg-named-'));
|
||||
try {
|
||||
cpSync(src, dir, { recursive: true });
|
||||
rmSync(join(dir, '.codegraph'), { recursive: true, force: true });
|
||||
const cg = CodeGraph.initSync(dir);
|
||||
await cg.indexAll();
|
||||
cg.close?.();
|
||||
|
||||
console.log(`\ntail-render-ts · agent-named symbols must render\n`);
|
||||
for (const c of CASES) {
|
||||
const { response, results } = await runCase(dir, c);
|
||||
console.log(`── ${c.id} — ${c.why}`);
|
||||
console.log(` query "${c.query}"`);
|
||||
console.log(` response ${response.length.toLocaleString()} chars`);
|
||||
for (const r of results) {
|
||||
checked += 1;
|
||||
if (!r.rendered) failures += 1;
|
||||
console.log(` ${r.rendered ? 'PASS' : 'FAIL'} ${r.name} defined at ${r.file}:${r.line}`
|
||||
+ (r.rendered ? '' : ' — DEFINITION NOT IN RESPONSE'));
|
||||
}
|
||||
if (verbose && failures) {
|
||||
const spans = [...renderedLines(response)].sort((a, b) => a - b);
|
||||
console.log(` rendered lines: ${spans[0]}..${spans[spans.length - 1]} (${spans.length} lines)`);
|
||||
}
|
||||
console.log();
|
||||
}
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
console.log(failures === 0
|
||||
? `Every agent-named symbol rendered (${checked} checked).`
|
||||
: `${failures} of ${checked} agent-named symbols did NOT render.`);
|
||||
process.exit(failures === 0 ? 0 : 1);
|
||||
Reference in New Issue
Block a user