codegraph/scripts/agent-eval/probe-suite-envelope.mjs

#!/usr/bin/env node
/**
 * Deterministic 6-repo envelope sweep for `codegraph_explore` (CG-26).
 *
 * The allocation issues (CG-30 / CG-31 / CG-26) are all decided by how the
 * render loop divides a fixed byte ceiling, and the agent A/B is far too noisy
 * to see a 2K byte shift. This runs the SAME six queries the CG-30 and CG-31
 * benchmark tables use, against the same clean-rebuilt corpus indexes, and
 * prints the numbers those tables are made of: source chars delivered, files in
 * the final output, whether the hard ceiling cut anything, and whether the
 * epilogue survived.
 *
 * Numbers come from the CG-4 diagnostic (`CODEGRAPH_EXPLORE_DEBUG`), so this
 * measures the shipping allocator rather than re-deriving shares from markdown.
 *
 * Usage (needs a current `npm run build`, and full-REBUILT indexes — CG-33):
 *   node scripts/agent-eval/probe-suite-envelope.mjs
 *   node scripts/agent-eval/probe-suite-envelope.mjs --json > /tmp/new.json
 *   node scripts/agent-eval/probe-suite-envelope.mjs --baseline /tmp/base.json
 *   CORPUS=/tmp/codegraph-corpus node scripts/agent-eval/probe-suite-envelope.mjs
 */
import { mkdtempSync, readFileSync, rmSync, existsSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';

const CORPUS = process.env.CORPUS ?? '/tmp/codegraph-corpus';

/** The six suite repos + the exact queries the CG-30/CG-31 tables were measured on. */
const SUITE = [
  { id: 'django', q: 'How does a QuerySet turn into SQL and fetch rows from the database?' },
  { id: 'excalidraw', q: 'How does updating an element re-render the canvas on screen?' },
  { id: 'okhttp', q: 'How does a call go through the interceptor chain to the network?' },
  { id: 'tokio', q: 'How does a spawned task get scheduled and run by a worker?' },
  { id: 'gin', q: 'How does a registered route handler get invoked for an incoming HTTP request?' },
  { id: 'alamofire', q: 'How does a request get built and sent through the session?' },
];

const argv = process.argv.slice(2);
const asJson = argv.includes('--json');
const baselineAt = argv.includes('--baseline') ? argv[argv.indexOf('--baseline') + 1] : null;
const only = argv.filter((a) => !a.startsWith('--') && a !== baselineAt);

const say = (s = '') => { if (!asJson) console.log(s); };
const num = (n) => Math.round(n).toLocaleString('en-US');

const load = (rel) => import(pathToFileURL(resolve(rel)).href);
const idx = await load('dist/index.js');
const toolsMod = await load('dist/mcp/tools.js');
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
const ToolHandler = toolsMod.ToolHandler ?? toolsMod.default?.ToolHandler;
if (typeof CodeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') {
  console.error('could not resolve CodeGraph/ToolHandler from dist/ — run `npm run build`');
  process.exit(2);
}

const tmp = mkdtempSync(join(tmpdir(), 'cg-suite-'));
const results = [];
try {
  for (const { id, q } of SUITE) {
    if (only.length > 0 && !only.includes(id)) continue;
    const repo = join(CORPUS, id);
    if (!existsSync(join(repo, '.codegraph', 'codegraph.db'))) {
      say(`${id}: no index at ${repo} — skipped`);
      continue;
    }
    const sidecar = join(tmp, `${id}.jsonl`);
    process.env.CODEGRAPH_EXPLORE_DEBUG = sidecar;
    const cg = CodeGraph.openSync(repo);
    const h = new ToolHandler(cg);
    const res = await h.execute('codegraph_explore', { query: q });
    const text = res.content?.[0]?.text ?? '';
    try { cg.close?.(); } catch { /* best effort */ }
    const report = JSON.parse(readFileSync(sidecar, 'utf8').trim().split('\n').pop());
    results.push({
      repo: id,
      sourceChars: report.envelope.sourceChars,
      envelopeChars: report.envelope.chars,
      allocatedChars: report.envelope.allocatedChars,
      hardCeiling: report.budget.hardCeiling,
      truncated: report.envelope.truncated,
      files: report.selection.filesInFinalOutput,
      // Did the response keep its trailing pointer list / notes, or did the
      // hard ceiling spend them? This is CG-26's residual 1.
      epilogueCut: text.includes('omitted for size'),
      sectionCut: text.includes('output truncated to budget'),
      notShown: text.includes('Not shown above'),
      budgetNote: text.includes('advisory only, NOT a quota'),
    });
  }
} finally {
  rmSync(tmp, { recursive: true, force: true });
}

if (asJson) {
  console.log(JSON.stringify(results, null, 2));
} else {
  const base = baselineAt ? JSON.parse(readFileSync(baselineAt, 'utf8')) : null;
  const byRepo = new Map((base ?? []).map((r) => [r.repo, r]));
  say('repo         source     Δ        env      files  cut          epilogue');
  say('-'.repeat(74));
  for (const r of results) {
    const b = byRepo.get(r.repo);
    const delta = b ? (r.sourceChars - b.sourceChars) : null;
    const dStr = delta === null ? '' : (delta > 0 ? `+${num(delta)}` : num(delta));
    const cut = r.sectionCut ? 'section' : r.epilogueCut ? 'epilogue' : '—';
    const epi = [r.notShown ? 'not-shown' : null, r.budgetNote ? 'budget-note' : null]
      .filter(Boolean).join('+') || 'none';
    say(
      `${r.repo.padEnd(12)} ${num(r.sourceChars).padStart(7)} ${dStr.padStart(8)} `
      + `${num(r.envelopeChars).padStart(7)} ${String(r.files).padStart(5)}  ${cut.padEnd(12)} ${epi}`,
    );
  }
  if (base) {
    const lost = results.filter((r) => {
      const b = byRepo.get(r.repo);
      return b && (r.sourceChars < b.sourceChars || r.files < b.files);
    });
    say('');
    say(lost.length === 0
      ? 'No repo delivers less source or fewer files than the baseline.'
      : `REGRESSION: ${lost.map((r) => r.repo).join(', ')} deliver less than baseline.`);
  }
}