From fa7fb8d127f6c127cfbe31473e90dee999ac50c1 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 4 Aug 2026 02:13:46 -0500 Subject: [PATCH] fix(explore): spend the reservation instead of dropping it (CG-21, #1500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A file whose proportional reservation lands below its own size stopped rendering whole, and the fallback cluster render could leave most of that reservation unspent — the bytes were neither delivered nor redistributed. Found by CG-15's agent A/B on the express control: `lib/utils.js`, the top-ranked file, was reserved 3,870 chars and spent 583. The whole-file grace bound (reservation + a sliver) sat just under the file's 5,293 bytes, so the whole render was declined and three matched symbols became a stub. The source envelope fell 13,849 -> 9,241 against an UNCHANGED budget, and the agent Read the file back four times in 1 run of 3. Two levers, per the task's candidate fixes: - WHOLE_FILE_BUY_FRACTION: a reservation that already covers 60% of a file buys the whole file. Funded from ONE shared overshoot pool sized at 15% of the envelope, spent in rank order. Per-file funding is the version that fails, and it fails the same way the bug does — the merit test is a ratio, so several files qualify at once and N independent overshoots push the last section past the render ceiling. Measured on the payroll fixture: three files bought whole and `payslip_builder.go` was dropped entirely. A dropped section is strictly worse than a clustered one. - Reservation carry-forward: what a file cannot spend goes to the next file down, bounded by MAX_SHARE. Tracked as two running totals rather than a `spent` variable threaded through the render loop's dozen exit paths, so no path can forget to account, and symmetric — a buy that overshoots suppresses slack until a later under-spend covers it. Express reproducer: `lib/utils.js` 583 -> 6,268 whole, envelope 9,241 -> 14,505 on the same 13,000 budget. The `memory-budget.ts` exception CG-14 documented is RESOLVED rather than re-justified: it ships whole again at 5,672 (27.3%) while `src/mcp/tools.ts` rises to 52.6% — so the answer file wins the envelope AND no previously-unclipped file is clipped, which is CG-12's own acceptance criterion finally holding. Two hermetic fixtures added, one per lever, because nothing in the suite had this shape — which is how it shipped. Both mutation-tested: removing the buy arm reddens 3, removing the carry-forward reddens 2, and removing the funding guard reddens 4 (including payroll's dropped `payslip_builder.go`). Their `fixture shape` blocks are load-bearing: the gates pass vacuously if a target ever drifts inside the grace bound, so the window is asserted directly. Full suite green (2,868 passed); both #1500 regression fixtures pass. --- __tests__/explore-allocation-e2e.test.ts | 417 +++++++++++++++++++++++ src/mcp/tools.ts | 114 ++++++- 2 files changed, 527 insertions(+), 4 deletions(-) diff --git a/__tests__/explore-allocation-e2e.test.ts b/__tests__/explore-allocation-e2e.test.ts index 70e4d21..dff9d9e 100644 --- a/__tests__/explore-allocation-e2e.test.ts +++ b/__tests__/explore-allocation-e2e.test.ts @@ -111,6 +111,17 @@ async function explore(project: Project, query: string) { score: (file: string) => fileOf(file)?.score ?? 0, /** Chars of source the allocator RESERVED for it, before anything rendered. */ allowance: (file: string) => fileOf(file)?.allowance ?? 0, + /** Which render path the loop took: `whole`, `clusters`, `focused`, `skeleton`. */ + render: (file: string) => fileOf(file)?.render ?? null, + /** + * Rank the ranking pass gave it (1 = the file the response leads with, and + * the first the render loop reaches). Read off the record rather than from + * the position in `report.files`, which the report re-sorts by delivered + * bytes for legibility. + */ + rank: (file: string) => fileOf(file)?.rank ?? -1, + /** Total source bytes delivered across every rendered file. */ + sourceTotal: () => [...bytes.values()].reduce((sum, n) => sum + n, 0), /** Fraction of the WHOLE response this file's source occupies. */ share: (file: string) => (bytes.get(file) ?? 0) / (text.length || 1), shareUnder: (prefix: string) => { @@ -412,6 +423,412 @@ export function log(message: string): void { }); }); +// ── 1b. CG-21: a reservation below the file's size must not lose its bytes ── + +/** + * The shape CG-15's agent A/B found in the wild, and the one thing the suite + * above could not catch: a file whose reservation lands BELOW its own size. + * + * Express, `lib/utils.js` (5,293 B), the top-ranked file for + * "res.send Content-Type ETag generateETag setETag": + * + * | | baseline | CG-12 | + * |---|---|---| + * | delivered | 6,380 (46.1%) whole | **583 (7.7%) cluster stub** | + * | source envelope (13,000 budget) | 13,849 | **9,241** | + * + * It was reserved 3,870 and spent 583. The whole-file grace bound + * (`allowance + min(800, allowance * 0.15)` = 4,450) sits just under the file, + * so the whole-file render is declined; the fallback cluster render has three + * matched symbols to work with and emits a stub. The other 3,287 chars were + * neither delivered nor redistributed — **the pool shrank by a third against an + * unchanged budget**, and the agent Read the file back four times. + * + * Everything about that is invisible to the fixtures above, and to the payroll + * one: both SATURATE (`[over budget] [TRUNCATED]`, 23,599 of a 23,600 pool), + * so there is no unspent reservation to lose. This fixture is built to sit in + * the gap instead — a mid-sized top-ranked file with a THIN matched-symbol set, + * sized just above its reservation — which is the combination that has to hold + * for the defect to reproduce, and is why it shipped. + * + * The `fixture shape` block below is load-bearing, not scaffolding: every gate + * here passes vacuously if the target ever drifts small enough for the grace + * bound to cover it, so the window `0.6 × size <= reservation < size` is + * asserted directly. + */ +describe('CG-21 — a reservation under the file size still buys the file', () => { + // Names two symbols that live in ONE mid-sized file (the named-seed tier is + // what puts it at rank 0) while the rest of the terms pull in its peers, so + // the proportional split hands the target well under its own size. + const QUERY = 'generateEtag compileEtag send response body'; + const TARGET = 'src/http/etag.ts'; + const RESPONSE = 'src/http/response.ts'; + const APPLICATION = 'src/http/application.ts'; + + /** + * Bulk for the target: real, extractable symbols that match NOTHING in the + * query. They make the file BIG without making it more relevant — which is + * precisely how a file ends up reserved less than it is worth in bytes. Kept + * dense (4 lines each) so the file stays well inside `WHOLE_FILE_MAX_LINES` + * and the byte bound is the only thing that can decline the whole render. + */ + const inertFiller = (n: number) => ` +export function normalizeLedgerRow${n}(row: string[], fallback: string, separator: string): string[] { + const trimmed = row.map((cell) => cell.trim()).filter((cell) => cell.length > 0 && cell !== separator); + return trimmed.length > 0 ? trimmed : [fallback, separator, String(trimmed.length), 'ledger-row-${n}']; +} +`; + + /** + * The matched-symbol set, deliberately THIN and small. This is the second + * half of the shape: with only these two tiny functions to cluster around, + * the fallback render emits a few hundred chars and abandons the rest of the + * reservation. A file with a fat matched set would spend its allowance the + * ordinary way and never expose the bug. + */ + const TARGET_SOURCE = ` +/** ETag helpers. */ +export function generateEtag(body: string): string { + return '"' + body.length.toString(16) + '"'; +} + +export function compileEtag(setting: string): (body: string) => string { + return setting === 'strong' ? generateEtag : (body: string) => 'W/' + generateEtag(body); +} +${Array.from({ length: 27 }, (_, i) => inertFiller(i + 1)).join('')}`; + + const responseMethod = (name: string) => ` + public ${name}(body: string): string { + const etag = compileEtag(this.etagSetting)(body); + this.headers.set('etag', etag); + return body; + } +`; + + const RESPONSE_SOURCE = ` +import { compileEtag } from './etag'; + +/** The response object: sends a body and negotiates its representation. */ +export class ServerResponse { + private headers = new Map(); + private etagSetting = 'strong'; +${[ + 'send', + 'sendBody', + 'sendResponse', + 'writeBody', + 'endResponse', + 'json', + 'setResponseBody', + 'flushResponseBody', +].map(responseMethod).join('')} +} +`; + + let project: Project; + let run: Awaited>; + let targetSize = 0; + + beforeAll(async () => { + project = await buildProject('codegraph-alloc-cg21-', { + [TARGET]: TARGET_SOURCE, + [RESPONSE]: RESPONSE_SOURCE, + [APPLICATION]: ` +import { ServerResponse } from './response'; + +/** The application: routes a request and hands the response its body. */ +export class Application { + private routes = new Map string>(); + + public handleRequest(path: string, res: ServerResponse, body: string): string { + const route = this.routes.get(path); + return route ? route(res) : res.send(body); + } + + public registerResponseRoute(path: string, handler: (res: ServerResponse) => string): void { + this.routes.set(path, handler); + } +} +`, + 'src/http/request.ts': ` +/** The request object: carries the inbound body. */ +export class ServerRequest { + public constructor(public readonly body: string) {} + + public freshResponseBody(): string { + return this.body.trim(); + } +} +`, + 'src/util/logger.ts': ` +export function log(message: string): void { + console.log(message); +} +`, + }); + targetSize = fs.readFileSync(path.join(project.dir, TARGET), 'utf-8').length; + run = await explore(project, QUERY); + }, 120_000); + + afterAll(() => destroyProject(project)); + + describe('fixture shape', () => { + it('ranks the target first, on a matched set of only two symbols', () => { + // Rank 0 is what makes the loss expensive: this is the file the response + // leads with, and the one the agent Reads back when it arrives as a stub. + expect(run.rank(TARGET)).toBe(1); + }); + + it('sizes the target ABOVE its reservation but inside the buy window', () => { + // The whole assertion set below is vacuous outside this window, so it is + // pinned here rather than assumed: + // reservation >= size → the grace bound already covers it, and the + // buy rule is never consulted (express's other + // three queries look like this). + // reservation < 0.6×size → the shortfall is real, clustering is the + // right answer, and the carry-forward — not the + // buy rule — is what conserves the bytes. + const reserved = run.allowance(TARGET); + expect(reserved).toBeGreaterThan(0); + expect(reserved).toBeLessThan(targetSize); + expect(reserved / targetSize).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.WHOLE_FILE_BUY_FRACTION); + // ...and specifically OUTSIDE the grace bound, which is the pre-CG-21 + // rule. If grace alone could carry it, this fixture proves nothing. + const graceBound = reserved + Math.min( + EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_MAX, + Math.round(reserved * EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_FRACTION), + ); + expect(targetSize).toBeGreaterThan(graceBound); + }); + + it('keeps the target inside the whole-file LINE bound, so only bytes can gate it', () => { + // `WHOLE_FILE_MAX_LINES` (220 for a non-central file) is a separate gate + // that also declines a whole render. If the fixture ever crossed it the + // suite would go red for the wrong reason — and, worse, a genuine + // regression in the BYTE bound would be masked by it. + const lines = fs.readFileSync(path.join(project.dir, TARGET), 'utf-8').split('\n').length; + expect(lines).toBeLessThanOrEqual(220); + }); + }); + + describe('the reservation is spent', () => { + it('delivers the target WHOLE rather than as a cluster stub', () => { + // The headline. Pre-CG-21 this file rendered `clusters` and emitted a few + // hundred chars against a multi-thousand-char reservation. + expect(run.render(TARGET)).toBe('whole'); + }); + + it('spends more than the reservation, not a fraction of it', () => { + // Stated as bytes so it bites independently of the render-mode label: a + // build that renamed the whole path but still emitted a stub fails here. + // Express: 583 delivered against 3,870 reserved. + const delivered = run.bytes.get(TARGET) ?? 0; + expect(delivered).toBeGreaterThanOrEqual(targetSize); + expect(delivered).toBeGreaterThan(run.allowance(TARGET)); + }); + + it('leaves no rendered file both under its reservation and short of content', () => { + // The defect stated as an invariant, which is what makes it general rather + // than a re-assertion of the case above: a rendered file either SPENDS what + // it was promised, or it ran out of file. Express's `lib/utils.js` did + // neither — 583 delivered, 3,870 promised, 5,293 bytes of file sitting + // there — and the difference was dropped rather than redistributed, which + // is why the source envelope fell 13,849 → 9,241 on an unchanged budget. + // + // `response.ts` is the case the naive "spend the whole pool" version of + // this test gets wrong: it delivers 1,635 of a 5,292 reservation and that + // is CORRECT — the file is only 1,635 bytes. A pool cannot be spent past + // the content that exists to fill it. + for (const f of run.report.files) { + if (!f.render || (f.emittedChars ?? 0) === 0) continue; + const size = fs.readFileSync(path.join(project.dir, f.path), 'utf-8').length; + expect(f.emittedChars, `${f.path} spent its reservation or ran out of file`) + .toBeGreaterThanOrEqual(Math.min(f.allowance ?? 0, size)); + } + }); + + it('holds the hard ceiling while doing it', () => { + // The buy rule spends MORE than the reservation, so the bound that stops + // it running away has to be re-proved here and not inherited: the + // overshoot pool is finite, and the 25K inline cap is absolute — past it + // the host writes the result to a file the agent Reads back. + const budget = getExploreOutputBudget(project.cg.getFiles().length); + const hardCeiling = Math.min(Math.round(budget.maxOutputChars * 1.5), INLINE_CAP); + expect(run.text.length).toBeLessThanOrEqual(hardCeiling); + expect(run.text.length).toBeLessThan(INLINE_CAP); + }); + + it('still serves the peers — concentration, not a single-file response', () => { + // The over-correction control for this fixture. Buying the target whole + // must not eat the files below it: that is the trade the shared overshoot + // pool refuses (it dropped `payslip_builder.go` when funding was per-file). + const peers = [RESPONSE, APPLICATION].filter((f) => (run.bytes.get(f) ?? 0) > 0); + expect(peers.length).toBeGreaterThan(0); + }); + }); +}); + +/** + * The other half of CG-21, and the half the whole-file buy rule cannot reach. + * + * Buying the file whole only helps when the reservation has already covered + * most of it. Below that the shortfall is real — the file is several times its + * reservation, and clustering IS the right render — but the bytes it cannot + * spend still must not evaporate. Express, query "compileETag req.fresh": + * `lib/utils.js` was reserved 3,809 and spent 791; the 3,018 chars it left had + * to reach `lib/response.js` below it, which delivered 4,650 on a 1,895 + * reservation. + * + * So this fixture is deliberately the INVERSE of the one above: the leading + * file is far too big for the buy rule to fire, and the assertion is on the + * file BELOW it. Without this, `allowance = reserved` — the whole carry-forward + * deleted — passes every other test in this file. + */ +describe('CG-21 — an unspendable reservation flows to the next file down', () => { + // Names three tiny callables that all live in the SPRAWL file — the named-seed + // tier is what puts a file with almost no matched content at rank 1 — plus one + // term the absorber's methods carry, so it ranks second rather than cliffing. + const QUERY = 'renderStaticScene renderInteractiveScene renderNewElementScene paintSceneLayer'; + // Rank 1: a huge file the query names two symbols in. Its reservation cannot + // approach its size, so it clusters — and clusters thinly, because those two + // symbols are all it matched. + const SPRAWL = 'src/scene/sprawl.ts'; + // Rank 2: dense with matched symbols and bigger than any share it can be + // reserved, so it will absorb whatever the file above it leaves. + const ABSORBER = 'src/render/absorber.ts'; + + const inertBulk = (n: number) => ` +export function reconcileLedgerEntry${n}(rows: string[], fallback: string, separator: string): string[] { + const trimmed = rows.map((cell) => cell.trim()).filter((cell) => cell.length > 0 && cell !== separator); + return trimmed.length > 0 ? trimmed : [fallback, separator, String(trimmed.length), 'entry-${n}']; +} +`; + + // Long ENOUGH, in lines, that the absorber cannot ship whole (220 lines is the + // other whole-file gate). That matters: a file that renders whole ignores the + // per-file budget entirely, and this fixture is about a budget being spent. + const matchedPaint = (n: number) => ` + public paintSceneLayer${n}(canvas: string, scene: string, element: string): string { + const appState = this.appState.get('layer${n}') ?? scene; + const painted = canvas + '|' + appState + '|' + element; + const stamped = painted + '|layer-${n}'; + const merged = stamped + '|' + scene + '|' + element; + const settled = merged.split('|').filter((part) => part.length > 0).join('|'); + this.appState.set('layer${n}', settled); + if (settled.length === 0) { + return this.paint(scene, scene); + } + return this.paint(settled, scene); + } +`; + + let project: Project; + let run: Awaited>; + + beforeAll(async () => { + project = await buildProject('codegraph-alloc-cg21-carry-', { + [SPRAWL]: ` +import { Absorber } from '../render/absorber'; + +/** Scene sprawl: three one-line answers buried in a very large file. */ +export function renderStaticScene(scene: string): string { + return new Absorber().paint(scene, scene); +} + +export function renderInteractiveScene(scene: string): string { + return new Absorber().paint(scene, scene + ':interactive'); +} + +export function renderNewElementScene(scene: string): string { + return new Absorber().paint(scene, scene + ':new-element'); +} +${Array.from({ length: 90 }, (_, i) => inertBulk(i + 1)).join('')}`, + [ABSORBER]: ` +/** The renderer: many matched paint passes, all of them wanted. */ +export class Absorber { + private appState = new Map(); + + public paint(element: string, scene: string): string { + return element + '|' + scene; + } +${Array.from({ length: 20 }, (_, i) => matchedPaint(i + 1)).join('')} +} +${/* Inert tail: pushes the absorber FAR past its reservation so the whole-file + buy rule cannot fire on it either. Without this the absorber ships whole + and the fixture measures the buy rule a second time instead of the + carry-forward — which is exactly how it read on the first attempt. */ + Array.from({ length: 40 }, (_, i) => inertBulk(100 + i)).join('')}`, + 'src/util/logger.ts': ` +export function log(message: string): void { + console.log(message); +} +`, + }); + run = await explore(project, QUERY); + }, 120_000); + + afterAll(() => destroyProject(project)); + + it('leaves the leading file unable to spend its reservation', () => { + // The precondition. If the sprawl file ever spends its share, there is no + // slack, and the assertion below passes for no reason at all. + const spent = run.bytes.get(SPRAWL) ?? 0; + expect(spent).toBeGreaterThan(0); + expect(spent).toBeLessThan(run.allowance(SPRAWL)); + // ...and it is out of reach of the buy rule, so this is genuinely the + // carry-forward's case and not a second test of the fixture above. + const size = fs.readFileSync(path.join(project.dir, SPRAWL), 'utf-8').length; + expect(run.allowance(SPRAWL) / size).toBeLessThan(EXPLORE_ALLOCATION.WHOLE_FILE_BUY_FRACTION); + }); + + it('hands the shortfall to the file below, which spends past its own reservation', () => { + // The lever. Measured both ways on this fixture: with the carry-forward the + // absorber delivers 9,297 against a 7,455 reservation; with + // `allowance = reserved` it delivers 7,479 — its reservation and nothing + // more, while the sprawl file's 4,408 unspent chars are dropped. + // + // The 1.1 margin is not padding. A cluster section can land a few chars over + // the budget it was selected against (whole symbol ranges, never sliced + // mid-method), so "delivered > reserved" alone is true by ~24 chars even on + // the mutated build — a test that passes on the defect. + const delivered = run.bytes.get(ABSORBER) ?? 0; + expect(delivered).toBeGreaterThan(Math.round(run.allowance(ABSORBER) * 1.1)); + }); + + it('keeps the shortfall in the envelope instead of dropping it', () => { + // The same lever read off the response as a whole, which is the form the + // user actually feels: express's source envelope fell 13,849 → 9,241 on an + // unchanged 13,000 budget because nothing picked up what `lib/utils.js` + // could not spend. Here: 10,033 delivered with the carry-forward, 8,215 + // without. + // + // Stated against what a no-carry build could produce — the leader's actual + // spend plus the absorber's own reservation — so it stays a statement about + // the mechanism rather than a hard-coded byte count. + const noCarryCeiling = (run.bytes.get(SPRAWL) ?? 0) + Math.round(run.allowance(ABSORBER) * 1.05); + expect(run.sourceTotal()).toBeGreaterThan(noCarryCeiling); + }); + + it('bounds the borrowing — slack concentrates, it does not consume', () => { + // Carried slack is clamped to `MAX_SHARE` of the envelope, so an + // under-spending leader cannot hand the file below it the whole response. + // The bound is stated WITH the spine allowance (`SPINE_CEILING`, 1.5x) + // folded in: a flow-path cluster is deliberately allowed past the per-file + // share, and that predates CG-21 — writing the tighter bound here would + // make this test fail on a build with no defect in it. + const budget = getExploreOutputBudget(project.cg.getFiles().length); + const clamp = Math.max( + run.allowance(ABSORBER), + Math.round(budget.maxOutputChars * EXPLORE_ALLOCATION.MAX_SHARE), + ); + expect(run.bytes.get(ABSORBER) ?? 0).toBeLessThanOrEqual(Math.round(clamp * 1.5)); + // The anti-starvation half, and the one that would actually bite: the file + // that lent the slack still gets rendered. + expect(run.bytes.get(SPRAWL) ?? 0).toBeGreaterThan(0); + expect(run.text.length).toBeLessThan(INLINE_CAP); + }); +}); + // ── 2. Degenerate and diffuse result sets ─────────────────────────────────── describe('allocation on degenerate result sets', () => { diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 85a0e99..0417615 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -521,6 +521,45 @@ export const EXPLORE_ALLOCATION = { */ WHOLE_FILE_GRACE_FRACTION: 0.15, WHOLE_FILE_GRACE_MAX: 800, + /** + * A reservation that already covers this fraction of a file BUYS THE WHOLE + * FILE (CG-21), even though the file is bigger than the reservation. + * + * The grace above is calibrated as a *sliver* — it only rescues a file that + * essentially fits. Below it there is a hole the render loop cannot fill: + * express's `lib/utils.js` (5,293 B) was the TOP-ranked file, reserved 3,870, + * declined the whole-file render at a 4,450 grace bound, and then spent 583 on + * a three-symbol cluster render. The other 3,287 chars of its reservation were + * neither redistributed nor delivered — the envelope shrank by a third against + * an unchanged budget and the agent Read the file back four times. + * + * So the rule is not "does the file fit the reservation" but "has the + * reservation already bought most of the file": at 0.6 the loop pays at most + * two-thirds of a reservation extra to avoid losing the whole thing, and it + * spends bytes it was going to spend anyway on a file that already earned + * them. Below the fraction the shortfall is real — the file is several times + * its reservation, clustering is the right answer, and the carry-forward + * (`reservedSoFar`/`sourceSpent` in the render loop) hands whatever it cannot + * spend to the next file down. + */ + WHOLE_FILE_BUY_FRACTION: 0.6, + /** + * The buy rule's overshoot is funded from ONE pool for the whole response, + * sized as this fraction of the envelope — deliberately the same 15% as + * `WHOLE_FILE_GRACE_FRACTION`, one level up: the grace is a sliver of a + * FILE's reservation, this is a sliver of the RESPONSE's envelope. + * + * Per-file funding is the version that fails, and it fails the same way the + * bug being fixed does. The merit test is a RATIO, so wherever several files + * sit near it they all qualify, and N independent overshoots inflate the + * response until the render ceiling drops whatever is last. Measured on the + * #1500 payroll fixture: three files bought whole and `payslip_builder.go` — + * the file that computes the payslip the question asks about, rank #6 — was + * dropped entirely so three higher-ranked files could each ship their final + * sliver. A dropped section is strictly worse than a clustered one, so one + * shared pool, spent in rank order, is the bound that matters. + */ + WHOLE_FILE_BUY_OVERSHOOT_FRACTION: 0.15, } as const; /** One candidate file's allocation inputs, in final rank order. */ @@ -3818,6 +3857,32 @@ export class ToolHandler { // instead of a different symbol's code under the requested name. const staleRendered: string[] = []; const staleOmitted: string[] = []; + // Reservation carry-forward (CG-21). A reservation is a promise the render + // loop has to KEEP, not a cap it may quietly under-use: a file that cannot + // spend what it was given — thin matched-symbol set, unreadable, drifted off + // disk, skipped for the ceiling — must hand the difference DOWN the rank + // order, not drop it. Tracked as two running totals rather than a `spent` + // variable threaded through the dozen `continue`s below, so no exit path can + // forget to account: everything the loop has PROMISED so far, and everything + // it has actually EMITTED. Their gap is the slack the next file may add to + // its own reservation. + // + // Symmetric on the other side: a whole-file buy that overshoots makes + // `sourceSpent` outrun `reservedSoFar`, which suppresses slack until a later + // under-spend covers the debt. So the pool is conserved in both directions, + // and no file is ever cut BELOW the reservation it was promised. + let reservedSoFar = 0; + let sourceSpent = 0; + // Funding line for the whole-file BUY rule: the response's SOURCE may reach + // everything the allocator promised plus one bounded overshoot, and no more. + // Measured against the promise rather than `renderCeiling` on purpose — the + // ceiling sits 50% above the envelope and says nothing about who is owed + // what, so funding a buy from it just moves the shortfall to whichever file + // the loop reaches last. See WHOLE_FILE_BUY_OVERSHOOT_FRACTION. + const reservedTotal = [...allocation.allowances.values()].reduce((sum, n) => sum + n, 0); + const sourceCeiling = reservedTotal + Math.round( + budget.maxOutputChars * EXPLORE_ALLOCATION.WHOLE_FILE_BUY_OVERSHOOT_FRACTION, + ); for (const [filePath, group] of sortedFiles) { if (filesIncluded >= maxFiles) { @@ -3835,11 +3900,24 @@ export class ToolHandler { // bounded by it instead of by the flat per-file cap, which is what stops // allocation from following file size: a small weakly-relevant file no // longer ships whole while the strongly-relevant one is clipped. - const allowance = allocation.allowances.get(filePath); - if (allowance === undefined) { + const reserved = allocation.allowances.get(filePath); + if (reserved === undefined) { diag?.recordSkip(filePath, 'max-files'); continue; } + // What this file may actually spend: its own reservation PLUS whatever the + // files above it left on the table. Every render bound below reads this, + // never `reserved` — that is what makes the carry-forward reach the render + // paths instead of being bookkeeping. Slack flows to the next file in RANK + // order because that is the only file a single-pass loop can still pay; + // `MAX_SHARE` keeps that from turning a weak tail file into the response, + // so the allocator's share ceiling holds end-to-end and not just at + // reservation time. + const allowance = Math.min( + reserved + Math.max(0, reservedSoFar - sourceSpent), + Math.max(reserved, Math.round(budget.maxOutputChars * EXPLORE_ALLOCATION.MAX_SHARE)), + ); + reservedSoFar += reserved; const absPath = validatePathWithinRoot(projectRoot, filePath); if (!absPath || !existsSync(absPath)) { diag?.recordSkip(filePath, 'unreadable'); @@ -3974,6 +4052,7 @@ export class ToolHandler { : 'skeleton (signatures only — codegraph_explore a name for its full body; do NOT Read)'; lines.push(fileSectionHeader(filePath, `${names} · ${tag}`), '', '```' + lang, skel.join('\n'), '```', ''); totalChars += skel.join('\n').length + 120; + sourceSpent += skel.join('\n').length; // Always "clipped": the per-symbol view elides bodies by construction. diag?.recordRender(filePath, bodyIds.size > 0 ? 'focused' : 'skeleton', skel.join('\n').length, true); renderedFilePaths.push(filePath); @@ -4005,11 +4084,36 @@ export class ToolHandler { // reservation removes the swing without touching the rule's purpose (a small // file sliced is a lossy subset the agent just Reads in full anyway). const WHOLE_FILE_MAX_LINES = isCentralFile ? 280 : 220; - const WHOLE_FILE_MAX_CHARS = allowance + Math.min( + // Two bounds, whichever is larger (CG-21): + // GRACE — the reservation plus a sliver, for a file that essentially fits; + // BUY — the reservation already covers most of the file, so the rest is + // cheaper to ship than to lose. A file between the two used to + // fall through to clustering and then spend a FRACTION of its + // reservation, and the remainder was neither delivered nor + // redistributed. See WHOLE_FILE_BUY_FRACTION. + // + // The BUY arm is two independent tests, and keeping them apart is the whole + // design. MERIT reads `reserved` — did THIS file's own relevance earn most + // of itself? — so borrowed slack can never promote a weak file to whole. + // FUNDING reads the shared overshoot pool, so the bytes exist to pay for it. + // Slack still reaches the file through `allowance`: it raises the GRACE arm + // and shrinks what a buy has to borrow. + const graceBound = allowance + Math.min( EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_MAX, Math.round(allowance * EXPLORE_ALLOCATION.WHOLE_FILE_GRACE_FRACTION), ); - if (fileLines.length <= WHOLE_FILE_MAX_LINES && fileContent.length <= WHOLE_FILE_MAX_CHARS) { + // FUNDING, as one inequality: after this file ships whole, does the source + // still fit the promise-plus-overshoot line WITH every reservation below + // it left payable? `owedBelow` is what makes it a displacement guard + // rather than a size cap — a buy that fits the line only by spending a + // lower-ranked file's reservation is the trade that dropped + // `payslip_builder.go`, and it is refused here. Self-limiting: each buy + // grows `sourceSpent`, so the pool cannot be spent twice. + const owedBelow = Math.max(0, reservedTotal - reservedSoFar); + const buysWhole = fileContent.length <= graceBound + || (reserved >= fileContent.length * EXPLORE_ALLOCATION.WHOLE_FILE_BUY_FRACTION + && sourceSpent + fileContent.length + owedBelow <= sourceCeiling); + if (fileLines.length <= WHOLE_FILE_MAX_LINES && buysWhole) { const body = fileContent.replace(/\n+$/, ''); let wholeSection = exploreLineNumbersEnabled() ? numberSourceLines(body, 1) : body; const uniqSymbols = [...new Set( @@ -4034,6 +4138,7 @@ export class ToolHandler { } lines.push(wholeHeader, '', '```' + lang, wholeSection, '```', ''); totalChars += wholeSection.length + 200; + sourceSpent += wholeSection.length; diag?.recordRender(filePath, 'whole', wholeSection.length, false); renderedFilePaths.push(filePath); filesIncluded++; @@ -4406,6 +4511,7 @@ export class ToolHandler { lines.push(''); totalChars += fileSection.length + 200; + sourceSpent += fileSection.length; diag?.recordRender(filePath, 'clusters', fileSection.length, chosenIndices.size < clusters.length); renderedFilePaths.push(filePath); filesIncluded++;