From 7cbde95ce23b5e8870f30cfded2044b135c57e13 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Thu, 6 Aug 2026 03:45:28 -0500 Subject: [PATCH] fix(explore): pay every admitted file on every render path (CG-26) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The invariant this closes: every admitted file receives at least its reservation before any file draws on carry-forward slack. CG-30 bounded an oversize cluster member and CG-31 gave the cluster path a displacement guard; three holes were left, and each one starved a file that had been admitted, reserved and — in the worst case — rendered. 1. The whole-file arms had no displacement guard. BUY's fit test read `renderCeiling - totalChars` (everyone's room) while its source-space sibling refused the same trade, and GRACE was not fit-tested at all. okhttp's CallServerInterceptor.kt shipped 8,499 chars on a 5,964 funded ceiling and the rank-6 file below it delivered nothing. Both arms now test the render they actually produce against `fundedHeadroom`, and a whole render that does not fit falls through to clustering instead of skipping the file. 2. Every section was charged a flat 200 chars while a real header runs 300-500. The loop believed it had room it did not have — okhttp allocated 26,601 against a 24,400 ceiling — so the final truncation threw a fully-rendered section away. Sections are charged their real cost now, the owed-below arithmetic uses a per-file overhead estimated from the file's own symbols, and a marginal overrun trims the weakest cluster (or windows the last one into the room that is left) rather than skipping the file over a rounding difference. 3. `owedPayableBelow` held all-or-nothing. When the last admitted file's FULL reservation no longer fit, nothing was held for it: on the precise-query fixture the rank-5 file took 4,134 chars against a 2,948 reservation while rank 6 — admitted, reserved 2,539 — was left 4 chars and skipped. It now holds the remainder while that remainder is still worth a section (MIN_CHARS). And the epilogue is budgeted instead of discarded. The flat 600-char margin was neither the epilogue's size (1,064 gin, 1,788 django, 2,231 excalidraw) nor a bound on it, so four of six suite repos shipped with no pointer list and no reminders at all. The loop now reserves the epilogue's FLOOR — the one line that says an uncovered area exists, plus a pointer for every file whose bytes were deliberately withheld (CG-12) — and the rest is fitted to the room that actually remains, in priority order, entry by entry. Sized from the real strings; no constant was swept against the suite. Deterministic, same clean-rebuilt indexes, baseline = CG-31 tip: repo base source new source files ceiling django 20,791 20,878 6 -> 6 was discarding its epilogue tokio 21,521 21,607 5 -> 5 was discarding its epilogue okhttp 19,034 18,870 5 -> 6 +1 file delivered excalidraw 20,204 19,652 8 -> 8 keeps its pointer list gin 10,776 10,776 4 -> 4 byte-identical alamofire 11,662 11,662 2 -> 2 byte-identical No repo truncates any more and none loses a file. okhttp and excalidraw trade 164 and 552 source chars on their LAST-ranked file for the pointer list naming what the response could not cover — bytes the CG-31 tip only had because it over-filled a ceiling it mis-measured and then discarded the epilogue whole. Co-Authored-By: Claude Opus 5 --- .../explore-reservation-invariant.test.ts | 265 ++++++++++ scripts/agent-eval/allocation-fixtures.json | 41 +- scripts/agent-eval/probe-allocation.mjs | 22 + scripts/agent-eval/probe-suite-envelope.mjs | 124 +++++ src/mcp/tools.ts | 479 +++++++++++++----- 5 files changed, 796 insertions(+), 135 deletions(-) create mode 100644 __tests__/explore-reservation-invariant.test.ts create mode 100644 scripts/agent-eval/probe-suite-envelope.mjs diff --git a/__tests__/explore-reservation-invariant.test.ts b/__tests__/explore-reservation-invariant.test.ts new file mode 100644 index 0000000..4b23df3 --- /dev/null +++ b/__tests__/explore-reservation-invariant.test.ts @@ -0,0 +1,265 @@ +/** + * Regression fixture for CG-26 — the end-to-end reservation invariant. + * + * Every admitted file receives at least its reservation before any file draws + * on carry-forward slack. + * + * CG-30 bounded how far an oversize cluster member may overshoot and CG-31 gave + * the cluster path a displacement guard. This pins the invariant they jointly + * satisfy across EVERY render path — cluster, whole-file grace, whole-file BUY — + * and in BOTH directions: the top-ranked file when the files below it overspend, + * and an admitted lower-ranked file when the top one does. + * + * Two things CG-26 fixed are pinned here because nothing else can see them: + * + * - The whole-file arms were fit-tested against raw room before the ceiling, + * never against what was still owed below. A grace-sized file could take a + * pending file's reservation on its way to the ceiling; okhttp's + * `CallServerInterceptor.kt` shipped 8,499 chars on a 5,964 funded ceiling + * and the rank-6 file below it delivered nothing. + * - Every section was charged a flat 200 chars of overhead while a real header + * runs 300–500. The loop believed it had room it did not have (okhttp + * rendered 26,601 chars against a 24,400 ceiling), so the final truncation + * threw a fully-rendered section away — the same starvation, arriving after + * the guard had done its work. + * + * Shares the `displacement-ts` fixture: four pipeline stages competing for one + * envelope, the first a single ~20K function, padded past 500 indexed files so + * the response sits on the 24K tier where reservations genuinely saturate the + * ceiling. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; +import { attributeSourceBytes } from '../src/mcp/explore-diagnostics'; +import type { ExploreDiagnosticReport, ExploreDiagnosticFile } from '../src/mcp/explore-diagnostics'; + +const FIXTURE_SRC = path.join(__dirname, 'fixtures', 'displacement-ts'); +const FILLER_FILES = 520; + +/** The giant: one ~20K function. Ranks #1 under the spread query. */ +const GIANT = 'src/pipeline/ingest.ts'; + +/** + * Three shapes, so the invariant is tested from both sides: + * spread — every stage named; the giant ranks #1 and overspends downwards. + * tail — the stages BELOW the giant named; something small ranks #1 while + * the giant competes from underneath. This is the direction CG-31's + * fixture could not reach. + * precise — one symbol. The concentration case the guard must not flatten. + */ +const QUERIES = { + spread: 'ingestRecords normalizeRecords enrichRecords publishRecords', + tail: 'publishRecords sinkRecord PipelineRecord ingestRecords', + precise: 'ingestRecords', +} as const; +type Shape = keyof typeof QUERIES; + +interface Probe { + response: string; + report: ExploreDiagnosticReport; + bytes: Map; +} + +describe('CG-26 — no admitted file is starved, on any render path', () => { + let testDir: string; + let cg: CodeGraph; + const probes = {} as Record; + + /** Admitted = the allocator reserved bytes for it. */ + const admitted = (probe: Probe): ExploreDiagnosticFile[] => + probe.report.files.filter((f) => (f.allowance ?? 0) > 0); + const all = (): Probe[] => Object.values(probes); + + beforeAll(async () => { + testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cg26-')); + fs.cpSync(FIXTURE_SRC, testDir, { recursive: true }); + fs.rmSync(path.join(testDir, '.codegraph'), { recursive: true, force: true }); + + const filler = path.join(testDir, 'src', 'generated'); + fs.mkdirSync(filler, { recursive: true }); + for (let i = 0; i < FILLER_FILES; i++) { + fs.writeFileSync( + path.join(filler, `unit${i}.ts`), + `export const seed${i} = ${i};\n` + + `export function widget${i}(n: number): number {\n return n * ${i + 1} + seed${i};\n}\n`, + ); + } + + cg = CodeGraph.initSync(testDir); + await cg.indexAll(); + + const sidecar = path.join(testDir, 'explore-diag.jsonl'); + const previous = process.env.CODEGRAPH_EXPLORE_DEBUG; + process.env.CODEGRAPH_EXPLORE_DEBUG = sidecar; + try { + const handler = new ToolHandler(cg); + for (const [shape, query] of Object.entries(QUERIES) as [Shape, string][]) { + const result = await handler.execute('codegraph_explore', { query }); + const response = result.content?.[0]?.text ?? ''; + const written = fs.readFileSync(sidecar, 'utf-8').trim().split('\n').filter(Boolean); + probes[shape] = { + response, + report: JSON.parse(written[written.length - 1]!) as ExploreDiagnosticReport, + bytes: attributeSourceBytes(response), + }; + } + } finally { + if (previous === undefined) delete process.env.CODEGRAPH_EXPLORE_DEBUG; + else process.env.CODEGRAPH_EXPLORE_DEBUG = previous; + } + }, 180_000); + + afterAll(() => { + if (cg) cg.destroy(); + if (testDir && fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true, force: true }); + }); + + // ── Fixture shape — if these rot, the gates below mean nothing ───────────── + + describe('fixture shape', () => { + it('sits on the 24K tier, where the reservations saturate the ceiling', () => { + expect(cg.getStats().fileCount).toBeGreaterThanOrEqual(500); + for (const probe of all()) expect(probe.report.budget.maxOutputChars).toBe(24000); + }); + + it('exercises both directions — the giant ranks #1 in one shape and lower in another', () => { + // Which shape puts it where is the ranker's business and may move; that + // it lands on BOTH sides across the three is what makes the gates below + // test the invariant rather than one arrangement of it. + const ranks = all().map((p) => p.report.files.find((f) => f.path === GIANT)?.rank ?? -1); + expect(ranks).toContain(1); + expect(ranks.some((r) => r > 1)).toBe(true); + }); + + it('exercises both render paths — something ships whole, something clusters', () => { + const modes = new Set(all().flatMap((p) => p.report.files.map((f) => f.render))); + expect(modes).toContain('clusters'); + expect(modes).toContain('whole'); + }); + }); + + // ── The invariant ───────────────────────────────────────────────────────── + + describe('the reservation invariant', () => { + it('CG-26 GATE: no file on ANY render path emits past what was still free', () => { + // CG-31 pinned this for `clusters` only. The whole-file arms were fit- + // tested against `renderCeiling - totalChars`, which is everyone's room, + // not this file's — so a whole render could spend a reservation the loop + // had already promised further down. + for (const [shape, probe] of Object.entries(probes) as [Shape, Probe][]) { + const over = probe.report.files + .filter((f) => f.render !== null && f.render !== 'dropped' && f.funded !== null) + // +1 for the render loop's own rounding on a windowed cut. + .filter((f) => f.emittedChars > f.funded! + 1) + .map((f) => `${shape}/${f.path}: ${f.emittedChars} emitted of ${f.funded} funded (${f.render})`); + expect(over).toEqual([]); + } + }); + + it('CG-26 GATE: every admitted file is delivered, whatever its rank', () => { + for (const [shape, probe] of Object.entries(probes) as [Shape, Probe][]) { + for (const rec of admitted(probe)) { + expect(rec.skipped, `${shape}/${rec.path} skipped`).toBeNull(); + expect(probe.bytes.get(rec.path) ?? 0, `${shape}/${rec.path} bytes`).toBeGreaterThan(0); + } + } + }); + + it('CG-26 GATE: the rank-#1 file gets its reservation even when a file below overspends', () => { + // The direction CG-31's fixture could not reach: under `tail` the giant + // ranks below a small file and draws far past its own reservation from + // carry-forward slack. Rank #1 must still receive what it was promised + // (or its whole file, if that is less). + for (const [shape, probe] of Object.entries(probes) as [Shape, Probe][]) { + const top = admitted(probe).sort((a, b) => a.rank - b.rank)[0]; + if (!top) continue; + const onDisk = fs.statSync(path.join(testDir, top.path)).size; + expect(probe.bytes.get(top.path) ?? 0, `${shape}/${top.path}`) + .toBeGreaterThanOrEqual(Math.min(top.allowance!, onDisk) * 0.9); + } + }); + + it('and the gate above is not vacuous — a lower-ranked file does overspend', () => { + const overspenders = (probe: Probe) => admitted(probe) + .filter((f) => f.rank > 1 && f.emittedChars > f.allowance!); + expect(overspenders(probes.tail).length).toBeGreaterThan(0); + }); + }); + + // ── What the ceiling must no longer do ──────────────────────────────────── + + describe('the hard ceiling never throws a rendered section away', () => { + it('the render loop spends what it counts — nothing is allocated past the ceiling', () => { + // Sections used to be charged a flat 200 chars against a header that runs + // 300–500, so the loop over-filled and the final truncation dropped whole + // sections. `allocatedChars` is the pre-truncation length: it staying + // under the ceiling IS the accounting being exact. + for (const [shape, probe] of Object.entries(probes) as [Shape, Probe][]) { + expect(probe.report.envelope.allocatedChars, shape) + .toBeLessThanOrEqual(probe.report.budget.hardCeiling); + expect(probe.report.envelope.truncated, shape).toBe(false); + } + }); + + it('no file is rendered and then dropped', () => { + for (const probe of all()) { + expect(probe.report.files.filter((f) => f.render === 'dropped')).toEqual([]); + } + }); + + it('keeps the response inside the hard ceiling', () => { + for (const probe of all()) { + expect(probe.report.envelope.chars).toBeLessThanOrEqual(probe.report.budget.hardCeiling); + } + }); + }); + + // ── The epilogue is budgeted, not discarded ─────────────────────────────── + + describe('the epilogue the loop budgeted for is the epilogue it emits', () => { + it('a response that withheld files still says so, and says to explore not Read', () => { + // The flat 600-char margin was neither the epilogue's size nor a bound on + // it, so a saturated response shipped with no pointer list and no + // reminders at all. Whatever else is traded away, the agent must be told + // an uncovered area exists and that another explore reaches it. + for (const [shape, probe] of Object.entries(probes) as [Shape, Probe][]) { + const withheld = probe.report.files.some( + (f) => f.render === null || (probe.bytes.get(f.path) ?? 0) === 0); + if (!withheld) continue; + expect( + /Not shown above|omitted for size|codegraph_explore/.test(probe.response), + `${shape} withheld files without saying where to look`, + ).toBe(true); + } + }); + + it('never steers the agent to Read', () => { + for (const probe of all()) { + expect(/use (the )?Read|fall back to Read(?!ing those files)/i.test(probe.response)).toBe(false); + } + }); + }); + + // ── The thing the invariant must NOT become ─────────────────────────────── + + describe('concentration survives', () => { + it('a precise symbol query still puts the most source in the named file', () => { + const mine = probes.precise.bytes.get(GIANT) ?? 0; + expect(mine).toBeGreaterThan(0); + for (const [p, n] of probes.precise.bytes) { + if (p === GIANT) continue; + expect(mine, `${GIANT} vs ${p}`).toBeGreaterThan(n); + } + }); + + it('is not an even split — the named file outspends its equal share', () => { + const rec = probes.precise.report.files.find((f) => f.path === GIANT)!; + const even = probes.precise.report.budget.maxOutputChars / admitted(probes.precise).length; + expect(rec.emittedChars).toBeGreaterThan(even); + }); + }); +}); diff --git a/scripts/agent-eval/allocation-fixtures.json b/scripts/agent-eval/allocation-fixtures.json index a99d12e..4a35fcc 100644 --- a/scripts/agent-eval/allocation-fixtures.json +++ b/scripts/agent-eval/allocation-fixtures.json @@ -51,7 +51,9 @@ "internal/domain/**", "cmd/**" ], - "incidental": ["internal/gen/**"] + "incidental": [ + "internal/gen/**" + ] }, "assert": { "answerShareAtLeast": 0.55, @@ -122,14 +124,31 @@ "The assertions are therefore relative — answer-vs-incidental, not fixed percentages." ], "groups": { - "answer": ["src/mcp/**"], - "incidental": ["scripts/**"] + "answer": [ + "src/mcp/**" + ], + "incidental": [ + "scripts/**" + ] }, "assert": { - "answerShareAtLeast": 0.5, + "$answerShareComment": [ + "Denominated in DELIVERED SOURCE, not in the whole envelope (CG-26). The", + "envelope-denominated form of this gate moved for reasons that have nothing", + "to do with allocation: it fell when the epilogue stopped being discarded,", + "and it fell again when a fifth ADMITTED file finally got paid its", + "reservation instead of being dropped by the ceiling. Both are the", + "improvements this epic exists to make, and a gate that reads them as", + "regressions is measuring the denominator. The fixture's own rationale", + "already says the assertions are relative, answer-vs-incidental, not fixed", + "percentages. 0.5 is unchanged; only what it is a share OF." + ], + "answerShareOfSourceAtLeast": 0.5, "incidentalShareAtMost": 0.25, "topFileGroup": "answer", - "mustDeliverBytes": ["src/mcp/tools.ts"] + "mustDeliverBytes": [ + "src/mcp/tools.ts" + ] }, "baseline": { "measuredOn": "2026-08-03", @@ -182,6 +201,18 @@ "src/resolution/lru-cache.ts": 0.087 }, "verdict": "ALL FOUR GATES PASS. The afterCG30 verdict called parse-run.mjs over-RESERVED; it was not — its reservation is 4,314 in both arms. It was over-SPENDING: 8,548 chars, drawing on reservations belonging to files the render loop had not reached yet, which is the CG-31 defect. With the displacement guard it renders 4,314, tools.ts's identical 8,282 chars go from 35.0% to 35.9% of a response that no longer overruns, and lru-cache.ts (dropped as memory-budget.ts was on the CG-30 arm) delivers. Note what did NOT change: allocation. This fixture moved because the render loop stopped spending other files' bytes, not because anything was re-ranked." + }, + "afterCG26": { + "measuredOn": "2026-08-06", + "note": "24,952 delivered of 24,949 allocated, nothing truncated — against the CG-31 tip's 23,083 on the SAME clean full rebuild of this repo's index. tools.ts delivers 8,282 chars in BOTH arms: identical bytes, unchanged reservation, unchanged rank. Total delivered SOURCE 21,228 against 18,105.", + "delivered": { + "src/mcp/tools.ts": 0.334, + "scripts/agent-eval/parse-run.mjs": 0.174, + "src/mcp/explore-session-state.ts": 0.141, + "src/resolution/memory-budget.ts": 0.126, + "src/resolution/lru-cache.ts": 0.081 + }, + "verdict": "ALL GATES PASS. The one that changed shape is answerShareAtLeast → answerShareOfSourceAtLeast: on the envelope denominator the answer group reads 47.5% here against 51.0% at the CG-31 tip, and neither number is about allocation. tools.ts's bytes are byte-identical between the arms; what moved is that the response now delivers a FIFTH admitted file (memory-budget.ts, rank 4, paid its full 3,123-char reservation — the CG-31 tip rendered it and then let the hard ceiling drop the whole section) and keeps epilogue prose it used to discard. Answer/incidental separation is unchanged and strong: tools.ts 33.4% against parse-run.mjs's 17.4%, incidental 17.4% (down from 18.7%), top delivered file still tools.ts. Measured in delivered source the answer group is 55.5%." } } ] diff --git a/scripts/agent-eval/probe-allocation.mjs b/scripts/agent-eval/probe-allocation.mjs index 1386731..9d26293 100755 --- a/scripts/agent-eval/probe-allocation.mjs +++ b/scripts/agent-eval/probe-allocation.mjs @@ -150,6 +150,28 @@ function evaluate(fixture, report, text) { `answer ${pct(share('answer'))} delivered (${pct(allocated.get('answer') ?? 0)} allocated)`, ); } + // Same question against the SOURCE the response delivered rather than the + // whole envelope (CG-26). The envelope-denominated gate above moves whenever + // the response's prose does — the epilogue surviving instead of being + // discarded costs it a point, and every additional admitted file that gets + // paid dilutes it further — so it cannot tell "the answer was starved" from + // "everything else was also delivered". Allocation is about source bytes; + // measure it in source bytes. + if (want.answerShareOfSourceAtLeast !== undefined) { + const sourceBy = new Map(); + let totalSource = 0; + for (const f of report.files) { + const g = groupOf(f.path, groups); + sourceBy.set(g, (sourceBy.get(g) ?? 0) + f.finalChars); + totalSource += f.finalChars; + } + const answerSource = totalSource > 0 ? (sourceBy.get('answer') ?? 0) / totalSource : 0; + add( + `answer group takes >= ${pct(want.answerShareOfSourceAtLeast)} of DELIVERED SOURCE`, + answerSource >= want.answerShareOfSourceAtLeast, + `answer ${num(sourceBy.get('answer') ?? 0)} of ${num(totalSource)} source chars (${pct(answerSource)})`, + ); + } if (want.incidentalShareAtMost !== undefined) { add( `incidental group takes <= ${pct(want.incidentalShareAtMost)} of the envelope`, diff --git a/scripts/agent-eval/probe-suite-envelope.mjs b/scripts/agent-eval/probe-suite-envelope.mjs new file mode 100644 index 0000000..dd04aad --- /dev/null +++ b/scripts/agent-eval/probe-suite-envelope.mjs @@ -0,0 +1,124 @@ +#!/usr/bin/env node +/** + * Deterministic 6-repo envelope sweep for `codegraph_explore` (CG-26). + * + * The allocation issues (CG-30 / CG-31 / CG-26) are all decided by how the + * render loop divides a fixed byte ceiling, and the agent A/B is far too noisy + * to see a 2K byte shift. This runs the SAME six queries the CG-30 and CG-31 + * benchmark tables use, against the same clean-rebuilt corpus indexes, and + * prints the numbers those tables are made of: source chars delivered, files in + * the final output, whether the hard ceiling cut anything, and whether the + * epilogue survived. + * + * Numbers come from the CG-4 diagnostic (`CODEGRAPH_EXPLORE_DEBUG`), so this + * measures the shipping allocator rather than re-deriving shares from markdown. + * + * Usage (needs a current `npm run build`, and full-REBUILT indexes — CG-33): + * node scripts/agent-eval/probe-suite-envelope.mjs + * node scripts/agent-eval/probe-suite-envelope.mjs --json > /tmp/new.json + * node scripts/agent-eval/probe-suite-envelope.mjs --baseline /tmp/base.json + * CORPUS=/tmp/codegraph-corpus node scripts/agent-eval/probe-suite-envelope.mjs + */ +import { mkdtempSync, readFileSync, rmSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const CORPUS = process.env.CORPUS ?? '/tmp/codegraph-corpus'; + +/** The six suite repos + the exact queries the CG-30/CG-31 tables were measured on. */ +const SUITE = [ + { id: 'django', q: 'How does a QuerySet turn into SQL and fetch rows from the database?' }, + { id: 'excalidraw', q: 'How does updating an element re-render the canvas on screen?' }, + { id: 'okhttp', q: 'How does a call go through the interceptor chain to the network?' }, + { id: 'tokio', q: 'How does a spawned task get scheduled and run by a worker?' }, + { id: 'gin', q: 'How does a registered route handler get invoked for an incoming HTTP request?' }, + { id: 'alamofire', q: 'How does a request get built and sent through the session?' }, +]; + +const argv = process.argv.slice(2); +const asJson = argv.includes('--json'); +const baselineAt = argv.includes('--baseline') ? argv[argv.indexOf('--baseline') + 1] : null; +const only = argv.filter((a) => !a.startsWith('--') && a !== baselineAt); + +const say = (s = '') => { if (!asJson) console.log(s); }; +const num = (n) => Math.round(n).toLocaleString('en-US'); + +const load = (rel) => import(pathToFileURL(resolve(rel)).href); +const idx = await load('dist/index.js'); +const toolsMod = await load('dist/mcp/tools.js'); +const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph; +const ToolHandler = toolsMod.ToolHandler ?? toolsMod.default?.ToolHandler; +if (typeof CodeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') { + console.error('could not resolve CodeGraph/ToolHandler from dist/ — run `npm run build`'); + process.exit(2); +} + +const tmp = mkdtempSync(join(tmpdir(), 'cg-suite-')); +const results = []; +try { + for (const { id, q } of SUITE) { + if (only.length > 0 && !only.includes(id)) continue; + const repo = join(CORPUS, id); + if (!existsSync(join(repo, '.codegraph', 'codegraph.db'))) { + say(`${id}: no index at ${repo} — skipped`); + continue; + } + const sidecar = join(tmp, `${id}.jsonl`); + process.env.CODEGRAPH_EXPLORE_DEBUG = sidecar; + const cg = CodeGraph.openSync(repo); + const h = new ToolHandler(cg); + const res = await h.execute('codegraph_explore', { query: q }); + const text = res.content?.[0]?.text ?? ''; + try { cg.close?.(); } catch { /* best effort */ } + const report = JSON.parse(readFileSync(sidecar, 'utf8').trim().split('\n').pop()); + results.push({ + repo: id, + sourceChars: report.envelope.sourceChars, + envelopeChars: report.envelope.chars, + allocatedChars: report.envelope.allocatedChars, + hardCeiling: report.budget.hardCeiling, + truncated: report.envelope.truncated, + files: report.selection.filesInFinalOutput, + // Did the response keep its trailing pointer list / notes, or did the + // hard ceiling spend them? This is CG-26's residual 1. + epilogueCut: text.includes('omitted for size'), + sectionCut: text.includes('output truncated to budget'), + notShown: text.includes('Not shown above'), + budgetNote: text.includes('**Explore budget:'), + }); + } +} finally { + rmSync(tmp, { recursive: true, force: true }); +} + +if (asJson) { + console.log(JSON.stringify(results, null, 2)); +} else { + const base = baselineAt ? JSON.parse(readFileSync(baselineAt, 'utf8')) : null; + const byRepo = new Map((base ?? []).map((r) => [r.repo, r])); + say('repo source Δ env files cut epilogue'); + say('-'.repeat(74)); + for (const r of results) { + const b = byRepo.get(r.repo); + const delta = b ? (r.sourceChars - b.sourceChars) : null; + const dStr = delta === null ? '' : (delta > 0 ? `+${num(delta)}` : num(delta)); + const cut = r.sectionCut ? 'section' : r.epilogueCut ? 'epilogue' : '—'; + const epi = [r.notShown ? 'not-shown' : null, r.budgetNote ? 'budget-note' : null] + .filter(Boolean).join('+') || 'none'; + say( + `${r.repo.padEnd(12)} ${num(r.sourceChars).padStart(7)} ${dStr.padStart(8)} ` + + `${num(r.envelopeChars).padStart(7)} ${String(r.files).padStart(5)} ${cut.padEnd(12)} ${epi}`, + ); + } + if (base) { + const lost = results.filter((r) => { + const b = byRepo.get(r.repo); + return b && (r.sourceChars < b.sourceChars || r.files < b.files); + }); + say(''); + say(lost.length === 0 + ? 'No repo delivers less source or fewer files than the baseline.' + : `REGRESSION: ${lost.map((r) => r.repo).join(', ')} deliver less than baseline.`); + } +} diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index c57ae0f..ec9d9b8 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -795,6 +795,34 @@ function fileSectionHeader(filePath: string, suffix: string): string { : `${FILE_SECTION_PREFIX}${filePath}\`**`; } +/** Header of `codegraph_explore`'s trailing pointer list. */ +const POINTER_HEADER = '**Not shown above — explore these names for their source**'; +/** Most files the pointer list ever names one-per-line; the rest are a count. */ +const POINTER_MAX_FILES = 10; +/** + * One pointer line: the file plus enough symbol names to make it NAMEABLE in a + * follow-up explore. Capped — an un-capped list ran to ~1.9K on the #1500 + * fixture (12 generated CRUD symbols on one line), meta-text bought at the + * price of the source bytes this section exists to point away from. + */ +function pointerLineFor(filePath: string, nodes: readonly Node[]): string { + const POINTER_SYMBOLS = 6; + const named = nodes.filter((n) => n.kind !== 'import' && n.kind !== 'export'); + const pool = named.length > 0 ? named : nodes; + const shown = pool.slice(0, POINTER_SYMBOLS); + const more = pool.length - shown.length; + const symbols = shown.map((n) => `${n.name}:${n.startLine}`).join(', ') + + (more > 0 ? `, +${more} more` : ''); + return `- ${filePath}: ${symbols}`; +} +/** + * Emitted when the response was too full to carry ANY of its pointer list. It + * is the one line the epilogue floor is reserved for: the list itself can be + * traded away, but the agent must still be told that an uncovered area exists + * and that another explore — not a Read — is how to reach it. + */ +const EPILOGUE_LOST_NOTE = '> (Trailing pointer list omitted for size. The source above is complete and verbatim — treat it as already Read. For anything this call did not cover, run another codegraph_explore with the specific names rather than reading those files.)'; + /** * Per-file staleness banner emitted at the top of a tool response when the * file watcher has pending events for files referenced by the response. @@ -3989,13 +4017,39 @@ export class ToolHandler { lines.push('> The code below is the **verbatim, current on-disk source** of these files — re-read from disk on this call and line-numbered, byte-for-byte identical to what the Read tool returns. It is NOT a summary, outline, or stale cache. Treat each block as a Read you have already performed: do not Read a file shown here.'); lines.push(''); + // The response's absolute cap. It MUST stay under the host's inline + // tool-result limit (~25K chars): above it the result is externalized to a + // file the agent Reads back (a 35K vscode explore did exactly this in the + // n=4 A/B). + const hardCeiling = Math.min(Math.round(budget.maxOutputChars * 1.5), 25000); + // What the epilogue is OWED — the part of it the loop must not spend (CG-26). + // Not a flat margin: the old 600 was neither the epilogue's size (1,064 on + // gin, 2,231 on excalidraw) nor a bound on it, so the loop budgeted for a + // thing that did not exist and the response then discarded the whole + // epilogue to fit. The floor is what the epilogue owes the AGENT rather + // than what it costs us: + // - the one-line note that says an uncovered area exists (always), and + // - a pointer for every file whose source was deliberately WITHHELD. + // A cliffed file's bytes were traded away on the promise that the agent + // can still name it in a follow-up call (CG-12); if the ceiling then + // eats that name the trade was a silent drop. + // Everything above the floor — the rest of the pointer list, the reminders + // — is elastic and fitted to the room that is actually left, at the end of + // this method. Sized from the REAL strings, never tuned: a constant swept + // against the suite is what CG-30's record warns about. + const cliffPointerFloor = [...cliffedFiles] + .slice(0, POINTER_MAX_FILES) + .reduce((n, fp) => { + const g = fileGroups.get(fp); + return g ? n + pointerLineFor(fp, g.nodes).length + 1 : n; + }, cliffedFiles.size > 0 ? POINTER_HEADER.length + 2 : 0); + const epilogueFloor = EPILOGUE_LOST_NOTE.length + 2 + cliffPointerFloor; // Absolute stop for the render loop. Reservations already fit the envelope, so // this only catches their bounded overshoot (the whole-file grace, an oversize // first cluster) — and catches it HERE, where a file can be skipped cleanly and // a later one still render, instead of at the final truncation, which lops off - // whichever section happened to land last. Kept in sync with `hardCeiling` - // below; the margin covers the drift epilogue and the trailing notes. - const renderCeiling = Math.min(Math.round(budget.maxOutputChars * 1.5), 25000) - 600; + // whichever section happened to land last. + const renderCeiling = hardCeiling - epilogueFloor; // `flow.text` is PART of the response — it is prepended to `lines` to make // the final output — so the render loop has to spend against it, and it // never did. Counting it is what makes `renderCeiling` the ceiling it @@ -4066,6 +4120,33 @@ export class ToolHandler { const sourceCeiling = reservedTotal + Math.round( budget.maxOutputChars * EXPLORE_ALLOCATION.WHOLE_FILE_BUY_OVERSHOOT_FRACTION, ); + /** + * What a file's section costs BESIDES its source, in render space: the + * header (path + up to `maxSymbolsInFileHeader` symbol names) plus the code + * fence and the blank lines around them (CG-26). + * + * `EXPLORE_ALLOCATION.FILE_OVERHEAD` is the ALLOCATOR's constant — the flat + * 200 it charges each admitted file when it splits the envelope — and using + * it here too was a category error worth ~250 chars per pending file: the + * render loop then held back a file's reservation but not the header that + * reservation has to arrive under, so the last admitted file was left just + * short of the room it needed and skipped whole. Estimated from the file's + * own candidate symbols, which is what the header is actually built from. + */ + const overheadCache = new Map(); + const sectionOverhead = (filePath: string, nodes: readonly Node[]): number => { + const hit = overheadCache.get(filePath); + if (hit !== undefined) return hit; + const names = [...new Set( + nodes.filter((n) => n.kind !== 'import' && n.kind !== 'export') + .map((n) => `${n.name}(${n.kind})`), + )].slice(0, budget.maxSymbolsInFileHeader); + // header + blank, then ```lang / body / ``` / blank around the source. + const cost = fileSectionHeader(filePath, names.join(', ')).length + 2 + + (nodes[0]?.language?.length ?? 0) + 11; + overheadCache.set(filePath, cost); + return cost; + }; /** * How much of what is still owed BELOW `fileIndex` the response can actually * still PAY, in render-space chars (CG-31). @@ -4088,10 +4169,24 @@ export class ToolHandler { const owedPayableBelow = (fileIndex: number, budgetLeft: number): number => { let held = 0; for (let j = fileIndex + 1; j < sortedFiles.length; j++) { - const r = allocation.allowances.get(sortedFiles[j]![0]); + const path = sortedFiles[j]![0]; + const r = allocation.allowances.get(path); if (r === undefined) continue; - const need = r + EXPLORE_ALLOCATION.FILE_OVERHEAD; - if (held + need > budgetLeft) break; + const overhead = sectionOverhead(path, sortedFiles[j]![1].nodes); + const need = r + overhead; + if (held + need > budgetLeft) { + // PART of a reservation is still a delivered file (CG-26). Holding + // all-or-nothing zeroed the last admitted file whenever its full + // reservation no longer fit: on the precise-query fixture the rank-5 + // file took 4,134 chars against a 2,948 reservation while rank 6 — + // admitted, reserved 2,539 — was left 4 chars and skipped. Hold the + // remainder instead, but only while it is still worth a section: + // under MIN_CHARS a slice cannot hold one complete method, and a + // fragment forces the Read this tool exists to prevent. + const partial = budgetLeft - held; + if (partial >= EXPLORE_ALLOCATION.MIN_CHARS + overhead) held += partial; + break; + } held += need; } return held; @@ -4155,7 +4250,7 @@ export class ToolHandler { // `totalChars` lower, which raises `headroom` one-for-one, so the // carry-forward the `allowance` line grants is exactly the carry-forward // this bound funds. - const headroom = Math.max(0, renderCeiling - totalChars - EXPLORE_ALLOCATION.FILE_OVERHEAD); + const headroom = Math.max(0, renderCeiling - totalChars - sectionOverhead(filePath, group.nodes)); const fundedHeadroom = Math.max( Math.min(reserved, headroom), headroom - owedPayableBelow(fileIndex, Math.max(0, headroom - reserved)), @@ -4244,7 +4339,11 @@ export class ToolHandler { ranges: ExploreLineRange[]; /** Spans replaced by the back-reference. */ covered: ExploreLineRange[]; - /** Chars charged to `totalChars` on top of the body (fences, header). */ + /** + * Chars charged on top of the body by the ANTI-ABANDONMENT RESTORE path + * only (it re-splices a section after the loop and needs one number for + * it). The loop itself charges the real cost — see `sectionCost`. + */ overhead: number; mode: 'whole' | 'clusters' | 'focused' | 'skeleton'; clipped: boolean; @@ -4260,6 +4359,16 @@ export class ToolHandler { const ranges = folded ? [] : opts.ranges; const at = lines.length; lines.push(opts.header, ''); + // Charge what the section ACTUALLY costs, not a flat 200 (CG-26). A + // header carries the path plus up to `maxSymbolsInFileHeader` symbol + // names and routinely runs 300–500 chars, so the flat charge made the + // loop believe it had room it did not have: okhttp rendered 26,601 + // chars against a 24,400 ceiling and the final truncation threw a + // fully-rendered section away. Everything downstream is expressed in + // these units — `headroom`, `fundedHeadroom`, every fit test — so an + // under-count is not a rounding error, it funds a promise out of bytes + // that do not exist and starves whoever the loop reaches last. + totalChars += opts.header.length + 2; if (opts.covered.length > 0) { const pointer = formatBackReference( filePath, @@ -4273,7 +4382,8 @@ export class ToolHandler { } if (body.length > 0) { lines.push('```' + lang, body, '```', ''); - totalChars += body.length + opts.overhead; + // ```lang \n body \n ``` \n '' \n — exact, same as the header above. + totalChars += body.length + lang.length + 11; sourceSpent += body.length; newSourceChars += body.length; diag?.recordRender(filePath, opts.mode, body.length, opts.clipped || opts.covered.length > 0); @@ -4288,7 +4398,8 @@ export class ToolHandler { // bytes) because the record means "source the agent HAS", not "bytes // this call spent" — refreshing them keeps a long session from ageing // them out of the retained window and re-serving them for nothing. - totalChars += opts.overhead; + // (The header is already charged above; a fully-held section is the + // header plus the pointer and nothing else.) diag?.recordRender(filePath, 'backref', 0, false); diag?.recordDedup(filePath, coveredChars(opts.covered), opts.covered); noteEmitted(filePath, opts.covered, 0, fingerprint); @@ -4512,27 +4623,38 @@ export class ToolHandler { // `fundedHeadroom` / `owedPayableBelow` (CG-31). const owedBelow = Math.max(0, reservedTotal - reservedSoFar); // Third condition on the BUY arm only: it must also FIT. A whole render - // that overruns `renderCeiling` is skipped ENTIRELY a few lines below (the - // branch refuses to slice a file mid-method), so attempting a buy that - // cannot fit trades a clustered section for NO section — the same trade - // the funding pool exists to refuse, arriving by a different route. - // Failing the test here instead drops through to the cluster path, which - // is bounded by `headroom` and always renders something. + // that overruns the ceiling is skipped ENTIRELY (the branch refuses to + // slice a file mid-method), so attempting a buy that cannot fit trades a + // clustered section for NO section — the same trade the funding pool + // exists to refuse, arriving by a different route. Failing the test here + // instead drops through to the cluster path, which is bounded by + // `fundedHeadroom` and always renders something. // - // Only reachable on the 24K tiers, which is why the small-tier fixtures - // cannot see it: the funding line is `reservedTotal + 0.15 * envelope` - // (~27.2K when a medium repo saturates) while `renderCeiling` is - // `min(1.5 * envelope, 25000) - 600` = 24.4K — so funding can approve - // ~2.8K that the ceiling then refuses. At 13K the line is ~14.4K against a - // ceiling of 18.9K and the two cannot cross. + // Measured against `fundedHeadroom`, not against `renderCeiling - totalChars` + // (CG-26). The two differ by exactly the displacement term: room before + // the ceiling belongs to every file the loop has not reached yet, and + // this arm used to read the raw room while its source-space sibling + // (`owedBelow`, above) refused the same trade. Source-space alone was not + // enough — the funding line is `reservedTotal + 0.15 * envelope` (~27.2K + // when a medium repo saturates) while the render ceiling is ~24.2K, so a + // buy can clear `sourceCeiling` and still take its bytes out of a + // lower-ranked file's reservation on the way to the ceiling. Now both + // arms enforce the same inequality in their own units, and the invariant + // holds on every path. // - // The GRACE arm is deliberately left alone: a file within a sliver of its - // reservation that still does not fit is genuinely at the end of a full - // response, and that behaviour predates this fix. + // The GRACE arm keeps its own bound (a file within a sliver of its + // reservation) but is fit-tested on the render it actually produces, at + // the emission site below, so it cannot displace either. const buysWhole = fileContent.length <= graceBound || (reserved >= fileContent.length * EXPLORE_ALLOCATION.WHOLE_FILE_BUY_FRACTION && sourceSpent + fileContent.length + owedBelow <= sourceCeiling - && totalChars + fileContent.length + EXPLORE_ALLOCATION.FILE_OVERHEAD <= renderCeiling); + && fileContent.length <= fundedHeadroom); + // Set by the whole-file arm when it actually emits. A whole render that + // does not FIT no longer ends the file's turn (CG-26) — it falls through + // to the cluster path below, which is bounded by `fundedHeadroom` and + // renders something. Skipping outright was the trade the funding pool + // exists to refuse: a clustered section traded for no section at all. + let renderedWhole = false; if (fileLines.length <= WHOLE_FILE_MAX_LINES && buysWhole) { const body = fileContent.replace(/\n+$/, ''); const wholeRange: ExploreLineRange = { start: 1, end: body.split('\n').length }; @@ -4556,28 +4678,41 @@ export class ToolHandler { const staleSuffix = fileStale ? ' · ⚠ changed since last index sync — source below is current; the symbol list may be outdated' : ''; const wholeHeader = fileSectionHeader(filePath, (omitted > 0 ? `${headerNames.join(', ')}, +${omitted} more` : headerNames.join(', ')) + staleSuffix); - if (totalChars + wholeSection.length + 200 > renderCeiling) { - // Don't slice a whole file mid-method — a file that doesn't fit is - // skipped whole. Half a file forces the Read this is meant to prevent. + // The fit test, on the bytes this render ACTUALLY costs (the numbered + // body, after dedup) rather than on the raw file — and against + // `fundedHeadroom`, so a whole render can no more spend a pending + // file's reservation than a clustered one can (CG-26). Both whole-file + // arms come through here, which is what closes the invariant on the + // GRACE path: grace is measured against this file's own allowance and + // says nothing about whether the bytes are still there to spend. + // Two tests, and they are different questions. `fundedHeadroom` is the + // DISPLACEMENT bound — may these bytes be spent without taking a + // pending file's reservation. `sectionCost` is the CEILING bound — do + // the header, fences and body actually fit what is left. The second one + // is exact now that the loop charges real section costs. + const wholeCost = wholeHeader.length + 2 + wholeSection.length + lang.length + 11; + if (wholeSection.length <= fundedHeadroom && totalChars + wholeCost <= renderCeiling) { + emitFileSection({ + header: wholeHeader, + body: wholeSection, + // The whole file, minus any trailing blank lines the render trimmed. + ranges: ddWhole.parts.map((p) => p.range), + covered: ddWhole.covered, + overhead: 200, + mode: 'whole', + clipped: false, + fullBody: fullSection, + fullRanges: [wholeRange], + }); + if (fileStale) staleRendered.push(filePath); + renderedWhole = true; + } else { + // Doesn't fit whole — don't slice a whole file mid-method here; fall + // through and let the cluster path pick body-shaped pieces of it. anyFileTrimmed = true; - diag?.recordSkip(filePath, 'budget-whole-file'); - continue; } - emitFileSection({ - header: wholeHeader, - body: wholeSection, - // The whole file, minus any trailing blank lines the render trimmed. - ranges: ddWhole.parts.map((p) => p.range), - covered: ddWhole.covered, - overhead: 200, - mode: 'whole', - clipped: false, - fullBody: fullSection, - fullRanges: [wholeRange], - }); - if (fileStale) staleRendered.push(filePath); - continue; } + if (renderedWhole) continue; // Drifted file too big for the whole-file window (#1474): the cluster / // skeleton renders below would slice current bytes at indexed ranges — @@ -4586,11 +4721,9 @@ export class ToolHandler { // never render a possibly-wrong slice. if (fileStale) { staleOmitted.push(filePath); - lines.push( - fileSectionHeader(filePath, '⚠ changed on disk after the last index sync — source omitted (indexed line ranges no longer match, so a slice could show the wrong code). Read this file directly for current content; the change is picked up on that project\'s next index sync.'), - '', - ); - totalChars += 260; + const staleHeader = fileSectionHeader(filePath, '⚠ changed on disk after the last index sync — source omitted (indexed line ranges no longer match, so a slice could show the wrong code). Read this file directly for current content; the change is picked up on that project\'s next index sync.'); + lines.push(staleHeader, ''); + totalChars += staleHeader.length + 2; diag?.recordRender(filePath, 'stale-omitted', 0, true); continue; } @@ -5072,23 +5205,30 @@ export class ToolHandler { } // Emit chosen clusters in source order so the file reads top-to-bottom. - let fileSection = ''; - const allSymbols: string[] = []; - const sectionRanges: ExploreLineRange[] = []; - const coveredRanges: ExploreLineRange[] = []; - for (let i = 0; i < clusters.length; i++) { - if (!chosenIndices.has(i)) continue; - const cluster = clusters[i]!; - const section = renderedClusters.get(i)!; - const text = sectionText(section.parts); - if (text.length > 0) { - if (fileSection.length > 0) fileSection += GAP_MARKER; - fileSection += text; + // Assembled through a function because it may have to run more than once: + // the fit test below trims the weakest cluster and re-assembles rather + // than skipping the file (CG-26). + const assembleSection = (chosen: ReadonlySet) => { + let text = ''; + const symbols: string[] = []; + const ranges: ExploreLineRange[] = []; + const covered: ExploreLineRange[] = []; + for (let i = 0; i < clusters.length; i++) { + if (!chosen.has(i)) continue; + const cluster = clusters[i]!; + const section = renderedClusters.get(i)!; + const part = sectionText(section.parts); + if (part.length > 0) { + if (text.length > 0) text += GAP_MARKER; + text += part; + } + ranges.push(...section.parts.map((p) => p.range)); + covered.push(...section.covered); + symbols.push(...cluster.symbols); } - sectionRanges.push(...section.parts.map((p) => p.range)); - coveredRanges.push(...section.covered); - allSymbols.push(...cluster.symbols); - } + return { text, symbols, ranges, covered }; + }; + let assembled = assembleSection(chosenIndices); // A chosen cluster is a COMPLETE method-range — we never cut through a body, // and a shrunk cluster drops WHOLE members for the same reason. An oversize @@ -5105,42 +5245,86 @@ export class ToolHandler { // files (Session.swift in Alamofire) produced 3.4KB symbol lists // from cluster scoring + edge-source lines, dwarfing the per-file // body cap. Show top names by frequency, with a "+N more" tail. - const symbolCounts = new Map(); - for (const s of allSymbols) { - symbolCounts.set(s, (symbolCounts.get(s) ?? 0) + 1); - } - const sortedSymbols = [...symbolCounts.entries()] - .sort((a, b) => b[1] - a[1]) - .map(([name]) => name); - const headerCap = budget.maxSymbolsInFileHeader; - const headerSymbols = sortedSymbols.slice(0, headerCap); - const omittedCount = sortedSymbols.length - headerSymbols.length; - const headerSuffix = omittedCount > 0 - ? `${headerSymbols.join(', ')}, +${omittedCount} more` - : headerSymbols.join(', '); - const fileHeader = fileSectionHeader(filePath, headerSuffix); + const headerFor = (symbols: readonly string[]): string => { + const symbolCounts = new Map(); + for (const s of symbols) symbolCounts.set(s, (symbolCounts.get(s) ?? 0) + 1); + const sortedSymbols = [...symbolCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([name]) => name); + const headerSymbols = sortedSymbols.slice(0, budget.maxSymbolsInFileHeader); + const omittedCount = sortedSymbols.length - headerSymbols.length; + return fileSectionHeader(filePath, omittedCount > 0 + ? `${headerSymbols.join(', ')}, +${omittedCount} more` + : headerSymbols.join(', ')); + }; // Last stop before the hard ceiling. The reservation already bounded cluster // selection above, so reaching this means the bounded overshoot (an oversize // first cluster, taken whole rather than sliced mid-method) ran the response - // out of room. Skip the file whole and keep scanning — never slice mid-method. - // This used to compare against `maxOutputChars` and exempt "necessary" files, - // which is how arrival order decided the answer: whichever files ranked first - // spent the envelope, and everything after them was dropped on a cap they had - // no say in. Reservations replace that exemption — a file that earned bytes - // was already given them. - if (totalChars + fileSection.length + 200 > renderCeiling) { + // out of room. + // + // Exact, like the whole-file arm above (CG-26): header + fences + body, + // not body + a flat 200. The displacement half of the invariant is + // already enforced on the body itself (`bodyCap` / `SPINE_CEILING` read + // `fundedHeadroom`); this is the ceiling half. And because it is exact it + // now bites at the margin — a header runs 300–500 chars where the body + // budget assumed 200 — so an overrun TRIMS the weakest cluster and + // re-assembles instead of skipping the file whole. Skipping a file over a + // ~300-char accounting difference is starvation by rounding: the file was + // admitted, reserved and rendered, and would have delivered nothing. + // Only when the top-ranked cluster alone cannot fit is the file skipped — + // that one is never sliced mid-method. + let fileHeader = headerFor(assembled.symbols); + let chosenNow = chosenIndices; + const costOfSection = (header: string, body: string) => + header.length + 2 + (body.length > 0 ? body.length + lang.length + 11 : 0); + while (totalChars + costOfSection(fileHeader, assembled.text) > renderCeiling + && chosenNow.size > 1) { + // Weakest first: `rankedClusters` is best-first, so walk it backwards. + const trimmed = new Set(chosenNow); + for (let i = rankedClusters.length - 1; i >= 0; i--) { + const idx = rankedClusters[i]!.idx; + if (trimmed.has(idx)) { trimmed.delete(idx); break; } + } + chosenNow = trimmed; + assembled = assembleSection(chosenNow); + fileHeader = headerFor(assembled.symbols); + anyFileTrimmed = true; + } + // One cluster left and still over — by the header estimate's error, at + // most a few hundred chars. Re-render it INTO the room that is actually + // left rather than skip the file: the same whole-line windowing an + // oversize cluster already gets (CG-30), just against an exact bound. + // The header is built from the cluster's symbols, not its text, so + // re-rendering cannot move the target. + if (totalChars + costOfSection(fileHeader, assembled.text) > renderCeiling + && chosenNow.size === 1) { + const idx = [...chosenNow][0]!; + const room = renderCeiling - totalChars + - (fileHeader.length + 2 + lang.length + 11); + if (room > 0) { + const reshrunk = renderCluster(clusters[idx]!, room, room); + renderedClusters.set(idx, reshrunk); + anyClusterShrunk = anyClusterShrunk || reshrunk.shrunk; + assembled = assembleSection(chosenNow); + anyFileTrimmed = true; + } + } + if (totalChars + costOfSection(fileHeader, assembled.text) > renderCeiling) { anyFileTrimmed = true; diag?.recordSkip(filePath, 'budget-clusters'); continue; } + const fileSection = assembled.text; + const sectionRanges = assembled.ranges; + const coveredRanges = assembled.covered; // The undeduped render of the same clusters, needed only if this file ends // up fully back-referenced AND the whole call finds nothing new to say — // see `suppressedFallback`. Built lazily: on every other call it is dead // weight. const fullClusterParts = fileSection.length === 0 - ? clusters.flatMap((c, i) => (chosenIndices.has(i) ? buildSection(c) : [])) + ? clusters.flatMap((c, i) => (chosenNow.has(i) ? buildSection(c) : [])) : []; emitFileSection({ header: fileHeader, @@ -5151,7 +5335,7 @@ export class ToolHandler { mode: 'clusters', // Windowing an oversize member elides source too — reporting it as // unclipped would hide exactly the cut the diagnostic exists to show. - clipped: chosenIndices.size < clusters.length || anyClusterShrunk, + clipped: chosenNow.size < clusters.length || anyClusterShrunk, fullBody: sectionText(fullClusterParts), fullRanges: fullClusterParts.map((p) => p.range), }); @@ -5236,6 +5420,14 @@ export class ToolHandler { // CLIFFED file is source we deliberately withheld, so the list is forced on // whenever there is one: withholding a file's bytes is only cheap if the agent // can still name it in a follow-up call (CG-12). + // The epilogue's three blocks are BUILT here and FITTED below (CG-26) — + // they are not pushed straight into `lines` any more. The render loop + // budgets for the epilogue floor it committed to (`EPILOGUE_FLOOR`); what + // the response can afford above that floor is only known now, so the + // blocks are assembled against the room that actually remains, in priority + // order, instead of being emitted whole and then discarded whole. + const pointerEntries: string[] = []; + let pointerOmitted = 0; if (budget.includeAdditionalFiles || cliffedFiles.size > 0) { // Everything ranked that didn't render, in rank order — cliffed files first, // since they outrank whatever the file cap cut. (Indexing by `filesIncluded` @@ -5251,64 +5443,91 @@ export class ToolHandler { .filter(([fp, group]) => group.score < scoreFloor && !rankedPaths.has(fp)) .sort((a, b) => b[1].score - a[1].score); const remainingFiles = [...remainingRelevant, ...peripheralFiles]; - if (remainingFiles.length > 0) { - lines.push('**Not shown above — explore these names for their source**'); - lines.push(''); - // A pointer only has to make the file NAMEABLE in a follow-up call, so cap - // the symbols per line: an un-capped list ran to ~1.9K on the #1500 fixture - // (12 generated CRUD symbols on one line), meta-text bought at the price of - // the source bytes this section exists to point away from. - const POINTER_SYMBOLS = 6; - for (const [filePath, group] of remainingFiles.slice(0, 10)) { - const named = group.nodes.filter(n => n.kind !== 'import' && n.kind !== 'export'); - const shown = (named.length > 0 ? named : group.nodes).slice(0, POINTER_SYMBOLS); - const more = (named.length > 0 ? named : group.nodes).length - shown.length; - const symbols = shown.map(n => `${n.name}:${n.startLine}`).join(', ') - + (more > 0 ? `, +${more} more` : ''); - lines.push(`- ${filePath}: ${symbols}`); - } - if (remainingFiles.length > 10) { - lines.push(`- ... and ${remainingFiles.length - 10} more files`); - } + for (const [filePath, group] of remainingFiles.slice(0, POINTER_MAX_FILES)) { + pointerEntries.push(pointerLineFor(filePath, group.nodes)); } + pointerOmitted = Math.max(0, remainingFiles.length - pointerEntries.length); } - // Add completeness signal so agents know they don't need to re-read these files. + // Completeness signal so agents know they don't need to re-read these files. // On small projects the budget gates this off — but if we actually had to // trim or drop clusters, surface a brief note so the agent knows it can // still Read for more detail. - if (budget.includeCompletenessSignal) { - lines.push(''); - lines.push('---'); - lines.push(`> **Complete source for ${filesIncluded} files is included above — do NOT re-read them.** If your question also needs files/symbols listed under "Not shown above" (or any area this call didn't cover), make ANOTHER codegraph_explore targeting those names — it returns the same source with line numbers and is cheaper and more complete than reading. Reserve Read for a single specific line range explore can't surface.`); - } else if (anyFileTrimmed) { - lines.push(''); - lines.push(`> Some file sections were trimmed for size. For a specific symbol you still need, run another \`codegraph_explore\` (or \`codegraph_node\`) with its exact name — line-numbered source, cheaper and more complete than Read.`); - } + const completenessBlock: string[] = budget.includeCompletenessSignal + ? ['', '---', `> **Complete source for ${filesIncluded} files is included above — do NOT re-read them.** If your question also needs files/symbols listed under "Not shown above" (or any area this call didn't cover), make ANOTHER codegraph_explore targeting those names — it returns the same source with line numbers and is cheaper and more complete than reading. Reserve Read for a single specific line range explore can't surface.`] + : anyFileTrimmed + ? ['', `> Some file sections were trimmed for size. For a specific symbol you still need, run another \`codegraph_explore\` (or \`codegraph_node\`) with its exact name — line-numbered source, cheaper and more complete than Read.`] + : []; - // Add explore budget note based on project size + // Explore budget note based on project size. + let budgetBlock: string[] = []; if (budget.includeBudgetNote) { try { const stats = cg.getStats(); const callBudget = getExploreBudget(stats.fileCount); - lines.push(''); - lines.push(`> **Explore budget: ${callBudget} calls for this project (${stats.fileCount.toLocaleString()} files indexed).** Each call covers ~6 files; if your question spans more, spend your remaining calls on the uncovered area BEFORE falling back to Read — another explore is cheaper and more complete than reading those files. Synthesize once you've used ${callBudget}.`); + budgetBlock = ['', `> **Explore budget: ${callBudget} calls for this project (${stats.fileCount.toLocaleString()} files indexed).** Each call covers ~6 files; if your question spans more, spend your remaining calls on the uncovered area BEFORE falling back to Read — another explore is cheaper and more complete than reading those files. Synthesize once you've used ${callBudget}.`]; } catch { // Stats unavailable — skip budget note } } - // Final ceiling — an ABSOLUTE inline cap, not a multiple of the budget. The - // render loop renders necessary (named/spine) files even a bit past - // maxOutputChars and caps only incidental ones, so this is the last safety. - // It MUST stay under the host's inline tool-result limit (~25K chars): above - // that the result is externalized to a file the agent Reads back (a 35K - // vscode explore did exactly this in the n=4 A/B). So allow a little - // necessary overflow above the 24K budget, but hard-stop at 25K — never into - // externalize territory. - const output = flow.text + lines.join('\n'); + // FIT THE EPILOGUE (CG-26). Before this, the epilogue was emitted whole and + // then, on a saturated response, discarded whole by the hard ceiling — four + // of six suite repos shipped with no pointer list and no reminders at all, + // and the render loop had "budgeted" 600 chars for something that measures + // 1,064–2,231. Neither number was the real one, because the epilogue is not + // one thing: a fixed floor the loop reserves for (the cut note, plus a + // pointer for every file whose bytes were deliberately WITHHELD — CG-12 + // makes those names load-bearing) and an elastic tail that takes what is + // left. Assembled in priority order — the do-not-re-read reminder first, + // then pointers in rank order, then the budget note — and emitted in + // document order. + const roomFor = (block: readonly string[]): number => + block.reduce((n, s) => n + s.length + 1, 0); + let room = hardCeiling - (flow.text.length + lines.join('\n').length); - const hardCeiling = Math.min(Math.round(budget.maxOutputChars * 1.5), 25000); + const keepCompleteness = completenessBlock.length > 0 + && roomFor(completenessBlock) <= room; + if (keepCompleteness) room -= roomFor(completenessBlock); + + const pointerBlock: string[] = []; + if (pointerEntries.length > 0) { + const head = [POINTER_HEADER, '']; + let left = room - roomFor(head); + if (left >= 0) { + let taken = 0; + for (const entry of pointerEntries) { + // Every entry we do NOT take has to be confessed by the tail line, so + // the tail's cost is part of taking one less than all of them. + const dropped = pointerEntries.length - taken - 1 + pointerOmitted; + const tail = dropped > 0 ? roomFor([`- ... and ${dropped} more files`]) : 0; + if (entry.length + 1 + tail > left) break; + left -= entry.length + 1; + taken++; + } + if (taken > 0) { + pointerBlock.push(...head, ...pointerEntries.slice(0, taken)); + const dropped = pointerEntries.length - taken + pointerOmitted; + if (dropped > 0) pointerBlock.push(`- ... and ${dropped} more files`); + room -= roomFor(pointerBlock); + } + } + } + // Nothing of the pointer list survived, but there WAS one — say so, in the + // one line that carries its instruction forward. + const pointersLost = pointerEntries.length > 0 && pointerBlock.length === 0; + + const keepBudgetNote = budgetBlock.length > 0 && roomFor(budgetBlock) <= room; + if (keepBudgetNote) room -= roomFor(budgetBlock); + + lines.push(...pointerBlock); + if (keepCompleteness) lines.push(...completenessBlock); + if (keepBudgetNote) lines.push(...budgetBlock); + if (pointersLost && roomFor([EPILOGUE_LOST_NOTE, '']) <= room) { + lines.push('', EPILOGUE_LOST_NOTE); + } + + const output = flow.text + lines.join('\n'); let finalText: string; // The epilogue costs less than a file section, so it is cut FIRST (CG-31). // Dropping a trailing section throws away source the render loop had already