feat(explore): point at source this session already sent, don't send it twice (CG-18)
A later explore call re-served whatever it re-ranked, so on the #1500 report the 4th call spent its envelope on the spine the 1st call had already delivered. CG-17 recorded what was served; this acts on it. What a withheld span becomes is the whole design: a POINTER, never a silence. An insufficient-feeling response is what sends an agent to Read, and one or two of those early in a session teach it to abandon codegraph — so the replacement names the file, the symbols and the line spans, and says both that the source came from THIS conversation and that the file has not changed since. - Content fingerprint, not the drift flag, gates it. They answer different questions: two calls inside one drift window served the same current bytes, while a file edited AND re-synced between calls is never "stale" and yet the agent's copy is now wrong. An edited file re-emits in full. - Only a covered run of >= 8 lines is replaced, and a remainder under 160 chars folds into the pointer. Below those the pointer costs more than the source and the block reads as shredded — a fence holding `228\t` is a broken-looking response, which is the expensive failure. - The reclaimed bytes go to files the agent has NOT seen, two ways: a smaller `sourceSpent` hands slack down CG-21's carry-forward pool, and a fully back-referenced file gives up its maxFiles slot the way a cliffed one does. Within a file, the cluster shrink now reads the DEDUPED length, so it never drops new symbols to make room for source it isn't sending. - If dedup suppresses everything and nothing new takes its place, the top suppressed file is spliced back in whole. An all-pointer response is the shape that reads as "codegraph found nothing"; one re-served file is the cheaper mistake. Kill switch: CODEGRAPH_EXPLORE_DEDUP=0.
This commit is contained in:
Colby McHenry
@@ -0,0 +1,360 @@
|
||||
/**
|
||||
* Cross-call source dedup (CG-18).
|
||||
*
|
||||
* A later `codegraph_explore` call in a session must not re-send source an
|
||||
* earlier call already delivered — but every byte it withholds has to be
|
||||
* replaced by a POINTER, never a silence. That asymmetry is what this suite
|
||||
* guards, because the two failure directions cost wildly different amounts: a
|
||||
* duplicate range wastes a few thousand chars, while a response that reads as
|
||||
* "codegraph doesn't have it" costs a Read — and one or two of those early in a
|
||||
* session teach an agent to stop calling the tool at all.
|
||||
*
|
||||
* Three layers:
|
||||
* 1. the range algebra — what is withheld, and the thresholds that stop it
|
||||
* from shredding a block into slivers;
|
||||
* 2. the fingerprint gate — an edit between two calls must re-emit, since a
|
||||
* pointer to pre-edit source is worse than no dedup at all;
|
||||
* 3. the handler seam — a real second call against a real index: no duplicate
|
||||
* ranges, a pointer for everything withheld, the reclaimed budget spent on
|
||||
* source the agent has NOT seen, and never an all-pointer response.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import CodeGraph from '../src/index';
|
||||
import { ToolHandler } from '../src/mcp/tools';
|
||||
import { ExploreSessionState, type ExploreProjectState } from '../src/mcp/explore-session-state';
|
||||
import {
|
||||
EXPLORE_DEDUP,
|
||||
dedupeRange,
|
||||
fileFingerprint,
|
||||
formatBackReference,
|
||||
intersectRange,
|
||||
mergeRanges,
|
||||
servedRangesForFile,
|
||||
subtractRange,
|
||||
symbolsInSpans,
|
||||
} from '../src/mcp/explore-dedup';
|
||||
|
||||
const FIXTURE_SRC = path.join(__dirname, 'fixtures', 'payroll-go');
|
||||
const QUERY = 'how does payroll cycle create and calculate payslips?';
|
||||
const POINTER = 'Already sent earlier in this conversation';
|
||||
|
||||
/** A prior-state shaped like the session tracker's, for the algebra tests. */
|
||||
function prior(files: Array<{ path: string; ranges: Array<[number, number]>; fingerprint?: string }>): ExploreProjectState {
|
||||
return {
|
||||
projectRoot: '/repo',
|
||||
callCount: 1,
|
||||
responseBytes: 1000,
|
||||
calls: [{
|
||||
index: 1,
|
||||
projectRoot: '/repo',
|
||||
query: 'q',
|
||||
sourceBytes: 500,
|
||||
responseBytes: 1000,
|
||||
files: files.map((f) => ({
|
||||
path: f.path,
|
||||
ranges: f.ranges.map(([start, end]) => ({ start, end })),
|
||||
bytes: 500,
|
||||
fingerprint: f.fingerprint,
|
||||
})),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
describe('range algebra', () => {
|
||||
it('subtracts a held span out of the middle of an intended one', () => {
|
||||
expect(subtractRange({ start: 1, end: 100 }, [{ start: 20, end: 40 }]))
|
||||
.toEqual([{ start: 1, end: 19 }, { start: 41, end: 100 }]);
|
||||
});
|
||||
|
||||
it('subtracts held spans at either edge, and a full cover to nothing', () => {
|
||||
expect(subtractRange({ start: 10, end: 50 }, [{ start: 1, end: 20 }]))
|
||||
.toEqual([{ start: 21, end: 50 }]);
|
||||
expect(subtractRange({ start: 10, end: 50 }, [{ start: 30, end: 90 }]))
|
||||
.toEqual([{ start: 10, end: 29 }]);
|
||||
expect(subtractRange({ start: 10, end: 50 }, [{ start: 1, end: 90 }])).toEqual([]);
|
||||
});
|
||||
|
||||
it('intersects to exactly what both sides hold', () => {
|
||||
expect(intersectRange({ start: 10, end: 50 }, [{ start: 1, end: 20 }, { start: 45, end: 80 }]))
|
||||
.toEqual([{ start: 10, end: 20 }, { start: 45, end: 50 }]);
|
||||
expect(intersectRange({ start: 10, end: 50 }, [{ start: 60, end: 80 }])).toEqual([]);
|
||||
});
|
||||
|
||||
it('merges touching spans — two adjacent blocks are one block of source', () => {
|
||||
expect(mergeRanges([{ start: 5, end: 9 }, { start: 10, end: 12 }, { start: 40, end: 41 }]))
|
||||
.toEqual([{ start: 5, end: 12 }, { start: 40, end: 41 }]);
|
||||
});
|
||||
|
||||
it('emits ONLY the delta when a later call wants a wider window', () => {
|
||||
// Call 1 sent the method; call 2 wants the class around it.
|
||||
const { emit, covered } = dedupeRange({ start: 80, end: 200 }, [{ start: 100, end: 140 }]);
|
||||
expect(covered).toEqual([{ start: 100, end: 140 }]);
|
||||
expect(emit).toEqual([{ start: 80, end: 99 }, { start: 141, end: 200 }]);
|
||||
});
|
||||
|
||||
it('withholds nothing when the overlap is smaller than a chunk worth pointing at', () => {
|
||||
// Context padding and signature lines land here. Replacing them costs more
|
||||
// in pointer text than the source is worth, and shreds the block.
|
||||
const overlap = EXPLORE_DEDUP.MIN_COVERED_LINES - 1;
|
||||
const { emit, covered } = dedupeRange({ start: 1, end: 100 }, [{ start: 10, end: 10 + overlap - 1 }]);
|
||||
expect(covered).toEqual([]);
|
||||
expect(emit).toEqual([{ start: 1, end: 100 }]);
|
||||
});
|
||||
|
||||
it('leaves an untouched span exactly as it was', () => {
|
||||
expect(dedupeRange({ start: 1, end: 50 }, [{ start: 200, end: 400 }]))
|
||||
.toEqual({ emit: [{ start: 1, end: 50 }], covered: [] });
|
||||
expect(dedupeRange({ start: 1, end: 50 }, []))
|
||||
.toEqual({ emit: [{ start: 1, end: 50 }], covered: [] });
|
||||
});
|
||||
});
|
||||
|
||||
describe('the fingerprint gate', () => {
|
||||
const FP = fileFingerprint('package main\nfunc main() {}\n');
|
||||
|
||||
it('returns the spans a call served for a file whose bytes are unchanged', () => {
|
||||
const state = prior([{ path: 'a.go', ranges: [[1, 40], [60, 80]], fingerprint: FP }]);
|
||||
expect(servedRangesForFile(state, 'a.go', FP)).toEqual([{ start: 1, end: 40 }, { start: 60, end: 80 }]);
|
||||
});
|
||||
|
||||
it('returns NOTHING once the file has been edited — a pointer would be wrong', () => {
|
||||
const state = prior([{ path: 'a.go', ranges: [[1, 40]], fingerprint: FP }]);
|
||||
const edited = fileFingerprint('package main\nfunc main() { changed() }\n');
|
||||
expect(servedRangesForFile(state, 'a.go', edited)).toEqual([]);
|
||||
});
|
||||
|
||||
it('ignores a record that cannot prove what it served', () => {
|
||||
const state = prior([{ path: 'a.go', ranges: [[1, 40]] }]);
|
||||
expect(servedRangesForFile(state, 'a.go', FP)).toEqual([]);
|
||||
});
|
||||
|
||||
it('never crosses files, and is empty for an untracked session', () => {
|
||||
const state = prior([{ path: 'a.go', ranges: [[1, 40]], fingerprint: FP }]);
|
||||
expect(servedRangesForFile(state, 'b.go', FP)).toEqual([]);
|
||||
expect(servedRangesForFile(null, 'a.go', FP)).toEqual([]);
|
||||
});
|
||||
|
||||
it('distinguishes two files that hash the same prefix but differ in length', () => {
|
||||
expect(fileFingerprint('abc')).not.toBe(fileFingerprint('abcd'));
|
||||
expect(fileFingerprint('abc')).toBe(fileFingerprint('abc'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('the back-reference itself', () => {
|
||||
const covered = [{ start: 100, end: 240 }];
|
||||
|
||||
it('names the file, the span and the symbols, and says the copy is still good', () => {
|
||||
const text = formatBackReference('internal/x.go', covered, ['RunCycle', 'BuildPayslip'], { partial: false });
|
||||
expect(text).toContain('internal/x.go');
|
||||
expect(text).toContain('L100-240');
|
||||
expect(text).toContain('RunCycle, BuildPayslip');
|
||||
expect(text).toContain(POINTER);
|
||||
expect(text).toContain('unchanged on disk');
|
||||
});
|
||||
|
||||
it('never tells the agent to Read, in either shape', () => {
|
||||
for (const partial of [true, false]) {
|
||||
const text = formatBackReference('x.go', covered, ['A'], { partial });
|
||||
expect(text).toMatch(/do NOT Read/i);
|
||||
expect(text).not.toMatch(/\bRead (this|the) file (for|to)\b/i);
|
||||
expect(text).not.toMatch(/omitted|unavailable|could not/i);
|
||||
}
|
||||
});
|
||||
|
||||
it('says the block below is only the NEW lines when the call still sends some', () => {
|
||||
expect(formatBackReference('x.go', covered, [], { partial: true })).toContain('NEW lines');
|
||||
expect(formatBackReference('x.go', covered, [], { partial: false })).toContain('not repeated here');
|
||||
});
|
||||
|
||||
it('names only symbols that actually fall in the withheld spans', () => {
|
||||
const nodes = [
|
||||
{ name: 'InSpan', kind: 'function', startLine: 110, endLine: 130 },
|
||||
{ name: 'Outside', kind: 'function', startLine: 300, endLine: 320 },
|
||||
{ name: 'AnImport', kind: 'import', startLine: 105, endLine: 105 },
|
||||
];
|
||||
expect(symbolsInSpans(nodes, covered)).toEqual(['InSpan']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('a second call against a real index', () => {
|
||||
let testDir: string;
|
||||
let cg: CodeGraph;
|
||||
let handler: ToolHandler;
|
||||
|
||||
beforeAll(async () => {
|
||||
testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cg18-'));
|
||||
fs.cpSync(FIXTURE_SRC, testDir, { recursive: true });
|
||||
fs.rmSync(path.join(testDir, '.codegraph'), { recursive: true, force: true });
|
||||
cg = CodeGraph.initSync(testDir);
|
||||
await cg.indexAll();
|
||||
handler = new ToolHandler(cg);
|
||||
}, 120_000);
|
||||
|
||||
afterAll(() => {
|
||||
if (cg) cg.destroy();
|
||||
if (testDir && fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const explore = (query: string, session?: ExploreSessionState, args: Record<string, unknown> = {}) =>
|
||||
handler.execute('codegraph_explore', { query, ...args }, session).then((r) => r.content[0]!.text);
|
||||
|
||||
/**
|
||||
* The line numbers actually inside each file's fenced source. Read off the
|
||||
* RESPONSE, not the bookkeeping — "no duplicate ranges" is a claim about what
|
||||
* the agent received, and checking it against the record we also wrote would
|
||||
* prove only that the two agree.
|
||||
*/
|
||||
function fencedLines(text: string): Map<string, Set<number>> {
|
||||
const out = new Map<string, Set<number>>();
|
||||
let current: string | null = null;
|
||||
let inFence = false;
|
||||
for (const line of text.split('\n')) {
|
||||
const header = /^\*\*`([^`]+)`\*\*/.exec(line);
|
||||
if (header && !inFence) { current = header[1]!; continue; }
|
||||
if (!inFence && current && line.startsWith('```')) { inFence = true; continue; }
|
||||
if (inFence && line === '```') { inFence = false; continue; }
|
||||
if (!inFence || !current) continue;
|
||||
const numbered = /^(\d+)\t/.exec(line);
|
||||
if (!numbered) continue;
|
||||
if (!out.has(current)) out.set(current, new Set());
|
||||
out.get(current)!.add(Number(numbered[1]));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
it('never re-sends a line it already sent, and points at every line it withholds', async () => {
|
||||
const session = new ExploreSessionState();
|
||||
const first = await explore(QUERY, session);
|
||||
const second = await explore(QUERY, session);
|
||||
|
||||
const before = fencedLines(first);
|
||||
const after = fencedLines(second);
|
||||
expect(after.size).toBeGreaterThan(0);
|
||||
|
||||
// Every file whose source the second call withheld carries a pointer, and
|
||||
// the pointer names it.
|
||||
expect(second).toContain(POINTER);
|
||||
for (const [file, lines] of before) {
|
||||
const repeated = [...(after.get(file) ?? [])].filter((n) => lines.has(n));
|
||||
if (repeated.length === 0) continue;
|
||||
// The only sanctioned repeat is the anti-abandonment restore, which fires
|
||||
// ONLY when the call found nothing new to say — and this one did.
|
||||
throw new Error(`call 2 re-sent ${file} lines ${repeated.slice(0, 5).join(',')}`);
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it('spends the reclaimed bytes on source the agent has not seen', async () => {
|
||||
const session = new ExploreSessionState();
|
||||
const first = await explore(QUERY, session);
|
||||
const second = await explore(QUERY, session);
|
||||
|
||||
const before = fencedLines(first);
|
||||
const after = fencedLines(second);
|
||||
const fresh = [...after.entries()].reduce(
|
||||
(sum, [file, lines]) => sum + [...lines].filter((n) => !(before.get(file)?.has(n))).length, 0);
|
||||
// Not merely "smaller": a shrunken response is what dedup must NOT produce.
|
||||
// The freed budget has to come back as lines the first call never sent.
|
||||
expect(fresh).toBeGreaterThan(20);
|
||||
expect(second.length).toBeLessThan(first.length);
|
||||
}, 120_000);
|
||||
|
||||
it('re-emits in full when the file changed between the two calls', async () => {
|
||||
const session = new ExploreSessionState();
|
||||
const target = path.join(testDir, 'internal/usecase/payroll/payslip_builder.go');
|
||||
const original = fs.readFileSync(target, 'utf-8');
|
||||
try {
|
||||
const first = await explore(QUERY, session);
|
||||
expect(fencedLines(first).has('internal/usecase/payroll/payslip_builder.go')).toBe(true);
|
||||
|
||||
fs.writeFileSync(target, original.replace('func sumKind(', 'func sumKindRenamed('), 'utf-8');
|
||||
const second = await explore(QUERY, session);
|
||||
|
||||
// The edited file is served again, whole — a pointer here would send the
|
||||
// agent to a copy of the file that no longer exists.
|
||||
const pointerLines = second.split('\n').filter((l) => l.includes(POINTER));
|
||||
expect(pointerLines.some((l) => l.includes('payslip_builder.go'))).toBe(false);
|
||||
expect(fencedLines(second).get('internal/usecase/payroll/payslip_builder.go')?.size ?? 0)
|
||||
.toBeGreaterThan(20);
|
||||
} finally {
|
||||
fs.writeFileSync(target, original, 'utf-8');
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it('always returns real source, even when the session already holds everything', async () => {
|
||||
const session = new ExploreSessionState();
|
||||
await explore(QUERY, session);
|
||||
await explore(QUERY, session);
|
||||
const third = await explore(QUERY, session);
|
||||
const fourth = await explore(QUERY, session);
|
||||
|
||||
// An all-pointer response is the shape that reads as failure. Every call
|
||||
// keeps at least one real fenced block, however much the session holds.
|
||||
for (const [n, text] of [[3, third], [4, fourth]] as const) {
|
||||
const lines = [...fencedLines(text).values()].reduce((s, set) => s + set.size, 0);
|
||||
expect(lines, `call ${n} returned no source at all`).toBeGreaterThan(10);
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
it('leaves the first call of a session untouched', async () => {
|
||||
const tracked = await explore(QUERY, new ExploreSessionState());
|
||||
const untracked = await explore(QUERY);
|
||||
expect(tracked).toBe(untracked);
|
||||
}, 120_000);
|
||||
|
||||
it('keeps two sessions on one handler independent', async () => {
|
||||
const a = new ExploreSessionState();
|
||||
const b = new ExploreSessionState();
|
||||
const firstForA = await explore(QUERY, a);
|
||||
await explore(QUERY, a);
|
||||
// B's first call has seen nothing, whatever A has been served.
|
||||
expect(await explore(QUERY, b)).toBe(firstForA);
|
||||
}, 180_000);
|
||||
|
||||
it('is off entirely under CODEGRAPH_EXPLORE_DEDUP=0', async () => {
|
||||
const session = new ExploreSessionState();
|
||||
const previous = process.env.CODEGRAPH_EXPLORE_DEDUP;
|
||||
process.env.CODEGRAPH_EXPLORE_DEDUP = '0';
|
||||
try {
|
||||
const first = await explore(QUERY, session);
|
||||
const second = await explore(QUERY, session);
|
||||
expect(second).toBe(first);
|
||||
expect(second).not.toContain(POINTER);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CODEGRAPH_EXPLORE_DEDUP;
|
||||
else process.env.CODEGRAPH_EXPLORE_DEDUP = previous;
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
it('reports the reclaimed bytes through the CG-4 diagnostic', async () => {
|
||||
const sidecar = path.join(testDir, 'cg18-diagnostic.jsonl');
|
||||
const session = new ExploreSessionState();
|
||||
const previous = process.env.CODEGRAPH_EXPLORE_DEBUG;
|
||||
process.env.CODEGRAPH_EXPLORE_DEBUG = sidecar;
|
||||
try {
|
||||
await explore(QUERY, session);
|
||||
await explore(QUERY, session);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CODEGRAPH_EXPLORE_DEBUG;
|
||||
else process.env.CODEGRAPH_EXPLORE_DEBUG = previous;
|
||||
}
|
||||
const [one, two] = fs.readFileSync(sidecar, 'utf-8').trim().split('\n').map((l) => JSON.parse(l));
|
||||
|
||||
expect(one.dedup.savedChars).toBe(0);
|
||||
expect(two.dedup.savedChars).toBeGreaterThan(1000);
|
||||
expect(two.dedup.backReferenced.length).toBeGreaterThan(0);
|
||||
|
||||
// The reclamation is legible file by file: a back-referenced file spent
|
||||
// none of its reservation, and the response still filled its envelope.
|
||||
const backref = two.files.filter((f: { render: string }) => f.render === 'backref');
|
||||
for (const f of backref) {
|
||||
expect(f.emittedChars).toBe(0);
|
||||
expect(f.dedupSavedChars).toBeGreaterThan(0);
|
||||
expect(f.dedupCovered.length).toBeGreaterThan(0);
|
||||
}
|
||||
const spentOnFreshSource = two.files.reduce((s: number, f: { emittedChars: number }) => s + f.emittedChars, 0);
|
||||
expect(spentOnFreshSource).toBeGreaterThan(0);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Cross-call source dedup for `codegraph_explore` (CG-18).
|
||||
*
|
||||
* The session record (CG-17) knows what earlier calls already sent. This module
|
||||
* is the algebra that turns that record into a decision for the call being
|
||||
* rendered: of the line ranges this call WOULD emit, which does the agent
|
||||
* already hold, and what is genuinely new.
|
||||
*
|
||||
* Three rules shape everything here, and all three come from the same place —
|
||||
* an insufficient-feeling response is what sends an agent to Read, and one or
|
||||
* two of those early in a session teach it to abandon codegraph entirely
|
||||
* (CLAUDE.md):
|
||||
*
|
||||
* 1. **A pointer, never a bare omission.** Removed source is replaced by a
|
||||
* back-reference naming the file, the symbols, and the line span, worded so
|
||||
* it is unmistakable that the source was already delivered IN THIS
|
||||
* CONVERSATION and is still current. Silence reads as "codegraph didn't
|
||||
* find it".
|
||||
* 2. **Only prove-it dedup.** A span is withheld only when the file's bytes
|
||||
* are byte-identical to what was served (a content fingerprint, not an
|
||||
* mtime and not the index's drift flag). An edit between calls means the
|
||||
* agent's copy is wrong, so the source is re-emitted in full.
|
||||
* 3. **Cut chunks, not slivers.** Only a covered run of at least
|
||||
* {@link EXPLORE_DEDUP.MIN_COVERED_LINES} lines is worth replacing. Below
|
||||
* that the pointer costs more than the source, and shattering a block into
|
||||
* one-line fragments produces exactly the ragged output that reads as a
|
||||
* failure. Everything not withheld is emitted — where the algebra is
|
||||
* unsure, it re-serves.
|
||||
*
|
||||
* Which way to be wrong, restated for this layer: re-serving something the agent
|
||||
* has is a few hundred wasted chars; withholding something it never saw is a
|
||||
* Read. Every threshold below leans to the first.
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
import type { ExploreLineRange, ExploreProjectState } from './explore-session-state';
|
||||
|
||||
export const EXPLORE_DEDUP = {
|
||||
/**
|
||||
* Shortest already-served run that may be replaced by a back-reference.
|
||||
*
|
||||
* Sized against what dedup is actually FOR — a later call re-serving a whole
|
||||
* method or file it already sent. A shorter covered run is either a signature
|
||||
* line in a skeleton render or the ±3 lines of context padding around a
|
||||
* cluster, and swapping either for a pointer trades bytes for noise: the
|
||||
* pointer sentence is itself ~140 chars, so under this length dedup would
|
||||
* make the response BIGGER while making it read as full of holes.
|
||||
*/
|
||||
MIN_COVERED_LINES: 8,
|
||||
/**
|
||||
* Below this many chars of NEW source, a file's remainder is folded into its
|
||||
* back-reference instead of being fenced on its own.
|
||||
*
|
||||
* The shape this exists for, seen on the CG-17 fixture: a third call whose
|
||||
* only unheld line was the file's trailing blank one, rendered as a code fence
|
||||
* containing `228\t`. A fence holding two lines of nothing reads as a broken
|
||||
* response, and reading as broken is the expensive failure — it is the thing
|
||||
* that sends an agent to Read and keeps it there. So a remainder this small is
|
||||
* dropped rather than shown. It is the one place this module withholds
|
||||
* something the agent has not seen, and it is bounded to ~two lines that sit
|
||||
* directly against source the agent does hold; the file is still named, with
|
||||
* its symbols, so one follow-up explore fetches it whole.
|
||||
*/
|
||||
MIN_DELTA_CHARS: 160,
|
||||
/** Line spans named in one pointer before it summarises the rest. */
|
||||
MAX_SPANS_IN_POINTER: 4,
|
||||
/** Symbols named in one pointer before it summarises the rest. */
|
||||
MAX_SYMBOLS_IN_POINTER: 5,
|
||||
} as const;
|
||||
|
||||
const OFF = new Set(['0', 'false', 'off', 'no']);
|
||||
|
||||
/**
|
||||
* Kill switch: `CODEGRAPH_EXPLORE_DEDUP=0` renders every call as if the session
|
||||
* had no history. Read per call (not memoized) so a test can toggle it.
|
||||
*/
|
||||
export function exploreDedupEnabled(): boolean {
|
||||
const raw = process.env.CODEGRAPH_EXPLORE_DEDUP;
|
||||
if (raw === undefined) return true;
|
||||
return !OFF.has(raw.trim().toLowerCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Identity of the bytes a call served for one file.
|
||||
*
|
||||
* This — not the index's drift flag — is what gates dedup. `isFileStaleOnDisk`
|
||||
* answers "did the file change since the last INDEX SYNC", which is a different
|
||||
* question with a different answer: two calls inside one drift window served the
|
||||
* same current bytes (dedup is correct), while a file edited and re-synced
|
||||
* between two calls is never "stale" and yet the agent's copy is now wrong
|
||||
* (dedup would be actively harmful). Length is prefixed so a hash prefix
|
||||
* collision cannot alias two files of different size.
|
||||
*/
|
||||
export function fileFingerprint(content: string): string {
|
||||
return `${content.length}:${createHash('sha1').update(content).digest('hex').slice(0, 16)}`;
|
||||
}
|
||||
|
||||
/** Sort + merge overlapping/adjacent spans into the smallest equivalent set. */
|
||||
export function mergeRanges(ranges: ReadonlyArray<ExploreLineRange>): ExploreLineRange[] {
|
||||
const valid = ranges
|
||||
.filter((r) => Number.isFinite(r.start) && Number.isFinite(r.end) && r.end >= r.start && r.start >= 1)
|
||||
.map((r) => ({ start: Math.floor(r.start), end: Math.floor(r.end) }))
|
||||
.sort((a, b) => a.start - b.start || a.end - b.end);
|
||||
const out: ExploreLineRange[] = [];
|
||||
for (const r of valid) {
|
||||
const last = out[out.length - 1];
|
||||
if (last && r.start <= last.end + 1) last.end = Math.max(last.end, r.end);
|
||||
else out.push({ ...r });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The parts of `range` that `served` covers. */
|
||||
export function intersectRange(
|
||||
range: ExploreLineRange,
|
||||
served: ReadonlyArray<ExploreLineRange>,
|
||||
): ExploreLineRange[] {
|
||||
const out: ExploreLineRange[] = [];
|
||||
for (const s of served) {
|
||||
const start = Math.max(range.start, s.start);
|
||||
const end = Math.min(range.end, s.end);
|
||||
if (end >= start) out.push({ start, end });
|
||||
}
|
||||
return mergeRanges(out);
|
||||
}
|
||||
|
||||
/** The parts of `range` that `cut` does NOT cover. */
|
||||
export function subtractRange(
|
||||
range: ExploreLineRange,
|
||||
cut: ReadonlyArray<ExploreLineRange>,
|
||||
): ExploreLineRange[] {
|
||||
const out: ExploreLineRange[] = [];
|
||||
let cursor = range.start;
|
||||
for (const c of mergeRanges(cut)) {
|
||||
if (c.end < cursor) continue;
|
||||
if (c.start > range.end) break;
|
||||
if (c.start > cursor) out.push({ start: cursor, end: Math.min(c.start - 1, range.end) });
|
||||
cursor = Math.max(cursor, c.end + 1);
|
||||
if (cursor > range.end) break;
|
||||
}
|
||||
if (cursor <= range.end) out.push({ start: cursor, end: range.end });
|
||||
return out;
|
||||
}
|
||||
|
||||
/** What one intended span becomes once the session's history is applied. */
|
||||
export interface RangeDedup {
|
||||
/** Spans to render now — everything not proven-already-held. */
|
||||
emit: ExploreLineRange[];
|
||||
/** Spans replaced by a back-reference. */
|
||||
covered: ExploreLineRange[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Split one intended span into what to emit and what to point back at.
|
||||
*
|
||||
* Covered runs shorter than {@link EXPLORE_DEDUP.MIN_COVERED_LINES} are left in
|
||||
* the emit set on purpose (rule 3 above) — so a span the agent holds "almost
|
||||
* all of" still comes back whole rather than as a stutter of fragments around
|
||||
* pointers.
|
||||
*/
|
||||
export function dedupeRange(
|
||||
range: ExploreLineRange,
|
||||
served: ReadonlyArray<ExploreLineRange>,
|
||||
minCovered: number = EXPLORE_DEDUP.MIN_COVERED_LINES,
|
||||
): RangeDedup {
|
||||
if (served.length === 0 || range.end < range.start) return { emit: [range], covered: [] };
|
||||
const covered = intersectRange(range, served).filter((r) => r.end - r.start + 1 >= minCovered);
|
||||
if (covered.length === 0) return { emit: [range], covered: [] };
|
||||
return { emit: subtractRange(range, covered), covered };
|
||||
}
|
||||
|
||||
/**
|
||||
* Every line span this session has already served for one file, but ONLY from
|
||||
* calls that served the SAME BYTES.
|
||||
*
|
||||
* A record with no fingerprint is ignored rather than trusted: it cannot prove
|
||||
* the agent's copy matches the file on disk now, and an unprovable match is
|
||||
* exactly the case where re-serving is right.
|
||||
*/
|
||||
export function servedRangesForFile(
|
||||
prior: ExploreProjectState | null,
|
||||
filePath: string,
|
||||
fingerprint: string,
|
||||
): ExploreLineRange[] {
|
||||
if (!prior) return [];
|
||||
const spans: ExploreLineRange[] = [];
|
||||
for (const call of prior.calls) {
|
||||
for (const file of call.files) {
|
||||
if (file.path !== filePath) continue;
|
||||
if (!file.fingerprint || file.fingerprint !== fingerprint) continue;
|
||||
spans.push(...file.ranges);
|
||||
}
|
||||
}
|
||||
return mergeRanges(spans);
|
||||
}
|
||||
|
||||
/** `L12`, `L12-40`, capped with a `+N more` tail. */
|
||||
export function formatSpans(spans: ReadonlyArray<ExploreLineRange>): string {
|
||||
const shown = spans.slice(0, EXPLORE_DEDUP.MAX_SPANS_IN_POINTER)
|
||||
.map((r) => (r.start === r.end ? `L${r.start}` : `L${r.start}-${r.end}`))
|
||||
.join(', ');
|
||||
const more = spans.length - EXPLORE_DEDUP.MAX_SPANS_IN_POINTER;
|
||||
return more > 0 ? `${shown}, +${more} more span${more === 1 ? '' : 's'}` : shown;
|
||||
}
|
||||
|
||||
/**
|
||||
* The line that replaces withheld source.
|
||||
*
|
||||
* It has one job: make the agent reach into its own context instead of into
|
||||
* Read. So it carries the three things needed to find the source it already has
|
||||
* — path, symbols, line spans — plus the two facts that make using it safe:
|
||||
* that it came from THIS conversation, and that the file has not changed since
|
||||
* (which is checked, not asserted — see {@link fileFingerprint}). It never says
|
||||
* "omitted", and it never steers to Read.
|
||||
*/
|
||||
export function formatBackReference(
|
||||
filePath: string,
|
||||
covered: ReadonlyArray<ExploreLineRange>,
|
||||
symbols: ReadonlyArray<string>,
|
||||
opts: { partial: boolean },
|
||||
): string {
|
||||
const names = symbols.slice(0, EXPLORE_DEDUP.MAX_SYMBOLS_IN_POINTER);
|
||||
const moreNames = symbols.length - names.length;
|
||||
const symbolPart = names.length > 0
|
||||
? ` (${names.join(', ')}${moreNames > 0 ? `, +${moreNames} more` : ''})`
|
||||
: '';
|
||||
const head = `> **Already sent earlier in this conversation:** \`${filePath}\` ${formatSpans(covered)}${symbolPart}`;
|
||||
const tail = opts.partial
|
||||
? ' — unchanged on disk since, so that copy is still exact. Only the NEW lines are shown below; scroll back for the rest. Do NOT Read this file.'
|
||||
: ' — unchanged on disk since, so that copy is still exact and is not repeated here. Use it from your context; do NOT Read this file.';
|
||||
return head + tail;
|
||||
}
|
||||
|
||||
/** Symbol names whose definitions fall inside the withheld spans. */
|
||||
export function symbolsInSpans(
|
||||
nodes: ReadonlyArray<{ name: string; kind: string; startLine: number; endLine: number }>,
|
||||
spans: ReadonlyArray<ExploreLineRange>,
|
||||
): string[] {
|
||||
const out: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const n of nodes) {
|
||||
if (n.kind === 'import' || n.kind === 'export') continue;
|
||||
if (!spans.some((s) => n.startLine <= s.end && (n.endLine || n.startLine) >= s.start)) continue;
|
||||
if (seen.has(n.name)) continue;
|
||||
seen.add(n.name);
|
||||
out.push(n.name);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -41,6 +41,7 @@ export type ExploreRenderMode =
|
||||
| 'focused' // per-symbol view, named/spine bodies full
|
||||
| 'skeleton' // per-symbol view, signatures only
|
||||
| 'stale-omitted' // drifted on disk; source deliberately withheld
|
||||
| 'backref' // fully served by an earlier call this session (CG-18)
|
||||
| 'dropped'; // rendered into `lines` but cut by the final hard ceiling
|
||||
|
||||
/** Why a ranked candidate never reached the output. */
|
||||
@@ -92,6 +93,17 @@ interface FileRecord extends ExploreCandidateMeta {
|
||||
*/
|
||||
allowance: number | null;
|
||||
render?: ExploreRenderMode;
|
||||
/**
|
||||
* Source chars this call did NOT re-send because an earlier call in the
|
||||
* session already did (CG-18). Reclaimed, not lost: it leaves through
|
||||
* `sourceSpent` (the carry-forward pool hands it to lower-ranked files) and,
|
||||
* for a fully back-referenced file, through the freed `maxFiles` slot. The
|
||||
* reallocation is legible as the difference between this file's
|
||||
* `allowance` and `emittedChars` against the files below it in the table.
|
||||
*/
|
||||
dedupSavedChars: number;
|
||||
/** Line spans replaced by a back-reference. */
|
||||
dedupCovered: Array<[number, number]>;
|
||||
/** Source chars the render loop handed to `lines` (pre-final-truncation). */
|
||||
emittedChars: number;
|
||||
/** Source chars present in the FINAL text — authoritative, truncation-aware. */
|
||||
@@ -130,6 +142,8 @@ export interface ExploreDiagnosticFile extends ExploreCandidateMeta {
|
||||
render: ExploreRenderMode | null;
|
||||
skipped: ExploreSkipReason | null;
|
||||
clipped: boolean;
|
||||
dedupSavedChars: number;
|
||||
dedupCovered: Array<[number, number]>;
|
||||
emittedChars: number;
|
||||
finalChars: number;
|
||||
share: number;
|
||||
@@ -191,6 +205,17 @@ export interface ExploreDiagnosticReport {
|
||||
filesRenderedByLoop: number;
|
||||
filesInFinalOutput: number;
|
||||
};
|
||||
/**
|
||||
* Cross-call source dedup (CG-18): what this call did NOT re-send because an
|
||||
* earlier call in this session already sent it, and where those bytes went.
|
||||
* `savedChars` 0 with a non-empty session block means nothing overlapped.
|
||||
*/
|
||||
dedup: {
|
||||
savedChars: number;
|
||||
backReferenced: string[];
|
||||
/** Files fully replaced by a pointer — each one also freed a `maxFiles` slot. */
|
||||
fullyBackReferenced: string[];
|
||||
};
|
||||
/** The proportional split (CG-12): what each file was promised, and why. */
|
||||
allocation: {
|
||||
/** Chars divided among admitted files (envelope minus per-file overhead). */
|
||||
@@ -338,6 +363,7 @@ export class ExploreDiagnostics {
|
||||
noteCandidate(path: string, meta: ExploreCandidateMeta): void {
|
||||
this.files.set(path, {
|
||||
path, ...meta, allowance: null,
|
||||
dedupSavedChars: 0, dedupCovered: [],
|
||||
emittedChars: 0, finalChars: 0, share: 0, allocatedShare: 0, clipped: false,
|
||||
});
|
||||
}
|
||||
@@ -375,6 +401,19 @@ export class ExploreDiagnostics {
|
||||
rec.skipped = undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Source this call withheld because the session already holds it (CG-18).
|
||||
* Called with `(path, 0, [])` to clear a record — the anti-abandonment restore
|
||||
* puts a suppressed file's source back, and a diagnostic still claiming the
|
||||
* saving would misreport where the envelope went.
|
||||
*/
|
||||
recordDedup(path: string, savedChars: number, covered: ReadonlyArray<{ start: number; end: number }>): void {
|
||||
const rec = this.files.get(path);
|
||||
if (!rec) return;
|
||||
rec.dedupSavedChars = savedChars;
|
||||
rec.dedupCovered = covered.map((r) => [r.start, r.end] as [number, number]);
|
||||
}
|
||||
|
||||
/**
|
||||
* A candidate was passed over before rendering. First reason wins — the
|
||||
* blanket `max-files` sweep must not overwrite a file's specific reason.
|
||||
@@ -410,8 +449,10 @@ export class ExploreDiagnostics {
|
||||
rec.share = envelope > 0 ? rec.finalChars / envelope : 0;
|
||||
rec.allocatedShare = allocatedChars > 0 ? rec.emittedChars / allocatedChars : 0;
|
||||
// Rendered into `lines` but absent from the final text → the hard
|
||||
// ceiling dropped its whole section.
|
||||
if (rec.render && rec.render !== 'stale-omitted' && rec.finalChars === 0) {
|
||||
// ceiling dropped its whole section. A back-referenced file has no
|
||||
// fenced source BY DESIGN (CG-18), so it is never "dropped".
|
||||
if (rec.render && rec.render !== 'stale-omitted' && rec.render !== 'backref'
|
||||
&& rec.finalChars === 0) {
|
||||
rec.render = 'dropped';
|
||||
rec.clipped = true;
|
||||
}
|
||||
@@ -467,6 +508,11 @@ export class ExploreDiagnostics {
|
||||
filesRenderedByLoop: filesIncluded,
|
||||
filesInFinalOutput: rendered.length,
|
||||
},
|
||||
dedup: {
|
||||
savedChars: records.reduce((s, r) => s + r.dedupSavedChars, 0),
|
||||
backReferenced: records.filter((r) => r.dedupSavedChars > 0).map((r) => r.path),
|
||||
fullyBackReferenced: records.filter((r) => r.render === 'backref').map((r) => r.path),
|
||||
},
|
||||
allocation: {
|
||||
pool: this.allocPool,
|
||||
cliffAt: round6(this.allocCliffAt),
|
||||
@@ -495,6 +541,8 @@ export class ExploreDiagnostics {
|
||||
render: r.render ?? null,
|
||||
skipped: r.skipped ?? null,
|
||||
clipped: r.clipped,
|
||||
dedupSavedChars: r.dedupSavedChars,
|
||||
dedupCovered: r.dedupCovered.map((s) => [...s] as [number, number]),
|
||||
emittedChars: r.emittedChars,
|
||||
finalChars: r.finalChars,
|
||||
share: round6(r.share),
|
||||
@@ -614,6 +662,15 @@ export function renderTable(report: ExploreDiagnosticReport): string {
|
||||
` relevance gate ${sel.graphGateApplied ? 'applied' : 'not applied'}` +
|
||||
` at graph >= ${sel.graphGateThreshold.toFixed(5)} (6% of max ${sel.maxGraph.toFixed(5)})`,
|
||||
);
|
||||
const dedup = report.dedup;
|
||||
if (dedup && dedup.savedChars > 0) {
|
||||
out.push(
|
||||
` dedup ${num(dedup.savedChars)} chars not re-sent` +
|
||||
` · ${dedup.fullyBackReferenced.length} file(s) fully back-referenced` +
|
||||
` (each also freed a maxFiles slot)` +
|
||||
(dedup.backReferenced.length > 0 ? `: ${dedup.backReferenced.join(', ')}` : ''),
|
||||
);
|
||||
}
|
||||
const alloc = report.allocation;
|
||||
out.push(
|
||||
` allocation ${num(alloc.reserved)} reserved of ${num(alloc.pool)} pool` +
|
||||
@@ -627,7 +684,7 @@ export function renderTable(report: ExploreDiagnosticReport): string {
|
||||
// Allocated (not delivered) is the allocator's own decision — the number the
|
||||
// budget work is about. Delivered is what the agent got. They differ only
|
||||
// when the ceiling truncated; showing both makes that divergence obvious.
|
||||
const shown = files.filter((f) => f.emittedChars > 0 || f.finalChars > 0);
|
||||
const shown = files.filter((f) => f.emittedChars > 0 || f.finalChars > 0 || f.render === 'backref');
|
||||
if (shown.length > 0) {
|
||||
out.push(' # alloc% deliv% bytes reserved score graph hits pen flags render file');
|
||||
for (const f of shown) {
|
||||
@@ -647,6 +704,11 @@ export function renderTable(report: ExploreDiagnosticReport): string {
|
||||
f.path,
|
||||
);
|
||||
out.push(' kinds: ' + (f.kinds || '-'));
|
||||
if (f.dedupSavedChars > 0) {
|
||||
const spans = f.dedupCovered.slice(0, 6).map(([a, b]) => (a === b ? `${a}` : `${a}-${b}`)).join(',');
|
||||
const more = f.dedupCovered.length > 6 ? `,+${f.dedupCovered.length - 6}` : '';
|
||||
out.push(` dedup: ${num(f.dedupSavedChars)} chars already sent this session · L${spans}${more}`);
|
||||
}
|
||||
}
|
||||
out.push(' (bytes = source allocated by the render loop; deliv% = 0 means the hard ceiling dropped the section)');
|
||||
out.push(' (* = clipped: some source in this file was elided, windowed, or its section dropped)');
|
||||
|
||||
@@ -70,6 +70,13 @@ export interface ExploreFileEmission {
|
||||
ranges: ExploreLineRange[];
|
||||
/** Source chars emitted for this file (excludes headers / fences). */
|
||||
bytes: number;
|
||||
/**
|
||||
* Identity of the bytes those ranges were sliced from (CG-18). Cross-call
|
||||
* dedup withholds a span only when the file still hashes to this, so an edit
|
||||
* between two calls re-serves instead of pointing at source the agent holds a
|
||||
* now-wrong copy of. Absent = unprovable, which dedup treats as "re-serve".
|
||||
*/
|
||||
fingerprint?: string;
|
||||
/** Set when ranges were dropped to stay under the per-file bound. */
|
||||
rangesTruncated?: boolean;
|
||||
}
|
||||
@@ -309,6 +316,7 @@ export class ExploreSessionState {
|
||||
.map((f) => {
|
||||
const { ranges, truncated } = coalesceRanges(f.ranges ?? []);
|
||||
const out: ExploreFileEmission = { path: f.path, ranges, bytes: Math.max(0, f.bytes || 0) };
|
||||
if (typeof f.fingerprint === 'string' && f.fingerprint) out.fingerprint = f.fingerprint;
|
||||
if (truncated) out.rangesTruncated = true;
|
||||
return out;
|
||||
});
|
||||
|
||||
@@ -62,6 +62,8 @@ calls; a grep/read exploration is dozens.
|
||||
- **After editing, check the staleness banner.** When a tool response starts with "⚠️ Some files referenced below were edited since the last index sync…", the listed files are pending re-index — Read those specific files for accurate content. Every file NOT in that banner is fresh, so still trust codegraph. A different, rarer banner — "⚠️ CodeGraph auto-sync is DISABLED…" — means live watching stopped entirely (the whole index is frozen, not just a few files); until it's resolved, Read files directly to confirm anything that may have changed.
|
||||
- **A file flagged "⚠ changed on disk after the last index sync" drifted from its index** (most common on projects queried via \`projectPath\`, which have no live watcher). Codegraph never serves a possibly-mis-sliced body from such a file — it either shows the file's full CURRENT source (trust it as a Read) or omits the source with this flag. When the source was omitted, Read that specific file; line numbers referencing it elsewhere in the response may be shifted until that project's next sync. All unflagged files remain trustworthy.
|
||||
|
||||
- **"Already sent earlier in this conversation" is a pointer, not a gap.** When a file's section carries that line instead of (or above) its source, an earlier \`codegraph_explore\` in THIS conversation already returned those exact lines and the file has not changed since — so the copy already in your context is current and exact. Scroll back to it; don't re-fetch it and don't Read the file. The bytes it freed went into source you have not seen yet, elsewhere in the same response.
|
||||
|
||||
## Limitations
|
||||
|
||||
- If a tool reports a project isn't indexed (no \`.codegraph/\`), stop calling codegraph tools for that project for the rest of the session and use your built-in tools there instead. Indexing is the user's decision — mention they can run \`codegraph init\` if it comes up, but don't run it yourself.
|
||||
|
||||
+383
-89
@@ -52,6 +52,16 @@ import {
|
||||
type ExploreFileEmission,
|
||||
type ExploreLineRange,
|
||||
} from './explore-session-state';
|
||||
import {
|
||||
EXPLORE_DEDUP,
|
||||
dedupeRange,
|
||||
exploreDedupEnabled,
|
||||
fileFingerprint,
|
||||
formatBackReference,
|
||||
mergeRanges,
|
||||
servedRangesForFile,
|
||||
symbolsInSpans,
|
||||
} from './explore-dedup';
|
||||
|
||||
/**
|
||||
* An expected, recoverable "codegraph can't serve this" condition — most
|
||||
@@ -3120,26 +3130,50 @@ export class ToolHandler {
|
||||
// byte-identical. It only OBSERVES: it must never feed back into rendering.
|
||||
const diag = ExploreDiagnostics.start(query, projectRoot, budget, maxFiles, indexedFileCount);
|
||||
|
||||
// What this session has already been served for THIS project (CG-17).
|
||||
// Read-only at this stage — it is reported in the diagnostic and nothing
|
||||
// else, so the response is unchanged. Cross-call dedup (CG-18) and budget
|
||||
// decay (CG-19) are the consumers this exists for.
|
||||
// What this session has already been served for THIS project (CG-17), and
|
||||
// whether this call may act on it (CG-18). Dedup is off on the session's
|
||||
// first call by construction — there is nothing to point back AT — and off
|
||||
// entirely under `CODEGRAPH_EXPLORE_DEDUP=0`.
|
||||
const priorCalls = viewForProject(readExploreSessionView(args), projectRoot);
|
||||
diag?.noteSession(priorCalls);
|
||||
const dedupEnabled = exploreDedupEnabled() && (priorCalls?.calls.length ?? 0) > 0;
|
||||
|
||||
// Cross-call dedup accounting (CG-18). `newSourceChars` is the load-bearing
|
||||
// one: a response whose source is ENTIRELY back-references is the shape that
|
||||
// reads as a failure, so the loop keeps the top suppressed file's real
|
||||
// section in hand and restores it if nothing new made it in — see
|
||||
// `suppressedFallback` below.
|
||||
let newSourceChars = 0;
|
||||
const backReferencedFiles: string[] = [];
|
||||
|
||||
// What this call ends up emitting, per file — the record handed back to the
|
||||
// session state on the main thread. Filled by every render path below, then
|
||||
// filtered to the files that SURVIVE the final hard-ceiling cut, so the
|
||||
// record is what the agent actually received rather than what the loop
|
||||
// hoped to send.
|
||||
const emittedByFile = new Map<string, { ranges: ExploreLineRange[]; bytes: number }>();
|
||||
const noteEmitted = (fp: string, ranges: ExploreLineRange[], bytes: number): void => {
|
||||
//
|
||||
// Back-referenced spans are recorded too, with zero bytes (CG-18): the
|
||||
// record means "source the agent HOLDS for this file", not "bytes this call
|
||||
// spent". Re-recording them refreshes them inside the retained-call window,
|
||||
// so a file pointed at across many calls doesn't age out of the history and
|
||||
// get re-served for no reason.
|
||||
const emittedByFile = new Map<
|
||||
string,
|
||||
{ ranges: ExploreLineRange[]; bytes: number; fingerprint?: string }
|
||||
>();
|
||||
const noteEmitted = (
|
||||
fp: string,
|
||||
ranges: ExploreLineRange[],
|
||||
bytes: number,
|
||||
fingerprint?: string,
|
||||
): void => {
|
||||
const existing = emittedByFile.get(fp);
|
||||
if (existing) {
|
||||
existing.ranges.push(...ranges);
|
||||
existing.bytes += bytes;
|
||||
if (fingerprint) existing.fingerprint = fingerprint;
|
||||
} else {
|
||||
emittedByFile.set(fp, { ranges: [...ranges], bytes });
|
||||
emittedByFile.set(fp, { ranges: [...ranges], bytes, fingerprint });
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3976,6 +4010,28 @@ export class ToolHandler {
|
||||
// instead of a different symbol's code under the requested name.
|
||||
const staleRendered: string[] = [];
|
||||
const staleOmitted: string[] = [];
|
||||
// Anti-abandonment hold-back (CG-18). The first file dedup suppressed
|
||||
// ENTIRELY, kept with its real section so it can be put back if the loop
|
||||
// ends with no new source anywhere. A response made only of pointers is the
|
||||
// shape that reads as "codegraph has nothing" — and one such response early
|
||||
// in a session is enough to make an agent stop calling the tool at all — so
|
||||
// the highest-ranked suppressed file is restored rather than risk it. It
|
||||
// costs a re-serve of one file, on the one call shape where dedup would
|
||||
// otherwise have saved everything: the safe direction.
|
||||
type SuppressedFallback = {
|
||||
filePath: string;
|
||||
/** Index in `lines` where this file's pointer block starts. */
|
||||
at: number;
|
||||
/** How many `lines` entries the pointer block occupies. */
|
||||
replacing: number;
|
||||
/** The full, undeduped section to splice back in. */
|
||||
section: string[];
|
||||
sourceChars: number;
|
||||
overhead: number;
|
||||
ranges: ExploreLineRange[];
|
||||
fingerprint: string;
|
||||
};
|
||||
let suppressedFallback: SuppressedFallback | null = null;
|
||||
// Reservation carry-forward (CG-21). A reservation is a promise the render
|
||||
// loop has to KEEP, not a cap it may quietly under-use: a file that cannot
|
||||
// spend what it was given — thin matched-symbol set, unreadable, drifted off
|
||||
@@ -4053,6 +4109,135 @@ export class ToolHandler {
|
||||
|
||||
const fileLines = fileContent.split('\n');
|
||||
const lang = group.nodes[0]?.language || '';
|
||||
const withLineNumbers = exploreLineNumbersEnabled();
|
||||
// Language-neutral separator between two non-contiguous slices of one file
|
||||
// (no `//` — not a comment in Python, Ruby, etc.). With line numbers on,
|
||||
// the line-number jump also signals the gap.
|
||||
const GAP_MARKER = '\n\n... (gap) ...\n\n';
|
||||
|
||||
// Cross-call dedup (CG-18). `served` is what THIS session already sent the
|
||||
// agent for THIS file, and it is empty unless the file still hashes to the
|
||||
// bytes those spans were sliced from — an edit between calls means the
|
||||
// agent's copy is wrong, so nothing is withheld. Every render path below
|
||||
// routes its spans through `dedupeSpans`, which is the only place a span
|
||||
// is ever dropped.
|
||||
const fingerprint = fileFingerprint(fileContent);
|
||||
const served = dedupEnabled ? servedRangesForFile(priorCalls, filePath, fingerprint) : [];
|
||||
/** Render one line span exactly as the render paths do. */
|
||||
const renderSpan = (r: ExploreLineRange): string => {
|
||||
const slice = fileLines.slice(r.start - 1, r.end).join('\n');
|
||||
return withLineNumbers ? numberSourceLines(slice, r.start) : slice;
|
||||
};
|
||||
/**
|
||||
* Apply the session history to a set of spans-with-text. A span the agent
|
||||
* already holds is dropped and reported in `covered`; a partially-held one
|
||||
* is re-rendered down to its new lines. Text is rebuilt from the surviving
|
||||
* spans rather than sliced out of the original string — the spans ARE the
|
||||
* contract with the session record, so rebuilding from them is what keeps
|
||||
* what we claim to have sent and what we sent the same thing.
|
||||
*/
|
||||
const dedupeSpans = (
|
||||
parts: ReadonlyArray<{ range: ExploreLineRange; text: string }>,
|
||||
): { parts: Array<{ range: ExploreLineRange; text: string }>; covered: ExploreLineRange[] } => {
|
||||
if (served.length === 0) return { parts: [...parts], covered: [] };
|
||||
const kept: Array<{ range: ExploreLineRange; text: string }> = [];
|
||||
const covered: ExploreLineRange[] = [];
|
||||
for (const part of parts) {
|
||||
const split = dedupeRange(part.range, served);
|
||||
if (split.covered.length === 0) {
|
||||
kept.push(part);
|
||||
continue;
|
||||
}
|
||||
covered.push(...split.covered);
|
||||
for (const r of split.emit) kept.push({ range: r, text: renderSpan(r) });
|
||||
}
|
||||
return { parts: kept, covered: mergeRanges(covered) };
|
||||
};
|
||||
const coveredChars = (spans: ReadonlyArray<ExploreLineRange>): number =>
|
||||
spans.reduce((sum, r) => sum + fileLines.slice(r.start - 1, r.end).join('\n').length, 0);
|
||||
/**
|
||||
* Emit one file's section — header, the back-reference for whatever the
|
||||
* agent already holds, and the fence for what is new. Every render path
|
||||
* ends here so that the dedup bookkeeping (freed bytes, freed `maxFiles`
|
||||
* slot, the session record, the diagnostic) is written in exactly one
|
||||
* place and no path can forget a piece of it.
|
||||
*
|
||||
* A fully-held file emits its header and pointer and NO fence, and
|
||||
* deliberately does not consume a `maxFiles` slot: that is half of where
|
||||
* the reclaimed budget goes (the other half is `sourceSpent`, which the
|
||||
* carry-forward pool hands down the rank order). Both send bytes to files
|
||||
* the agent has NOT seen, which is the whole point.
|
||||
*/
|
||||
const emitFileSection = (opts: {
|
||||
header: string;
|
||||
/** Deduped source. Empty ⇒ the agent already holds all of it. */
|
||||
body: string;
|
||||
/** Spans `body` covers. */
|
||||
ranges: ExploreLineRange[];
|
||||
/** Spans replaced by the back-reference. */
|
||||
covered: ExploreLineRange[];
|
||||
/** Chars charged to `totalChars` on top of the body (fences, header). */
|
||||
overhead: number;
|
||||
mode: 'whole' | 'clusters' | 'focused' | 'skeleton';
|
||||
clipped: boolean;
|
||||
/** The undeduped render, kept for the no-new-source fallback. */
|
||||
fullBody: string;
|
||||
fullRanges: ExploreLineRange[];
|
||||
}): void => {
|
||||
// A remainder too small to be worth a fence is folded into the pointer
|
||||
// (see MIN_DELTA_CHARS). Its ranges are then NOT recorded — the record
|
||||
// must only ever claim source that was actually sent.
|
||||
const folded = opts.covered.length > 0 && opts.body.length < EXPLORE_DEDUP.MIN_DELTA_CHARS;
|
||||
const body = folded ? '' : opts.body;
|
||||
const ranges = folded ? [] : opts.ranges;
|
||||
const at = lines.length;
|
||||
lines.push(opts.header, '');
|
||||
if (opts.covered.length > 0) {
|
||||
const pointer = formatBackReference(
|
||||
filePath,
|
||||
opts.covered,
|
||||
symbolsInSpans(group.nodes, opts.covered),
|
||||
{ partial: body.length > 0 },
|
||||
);
|
||||
lines.push(pointer, '');
|
||||
totalChars += pointer.length + 2;
|
||||
backReferencedFiles.push(filePath);
|
||||
}
|
||||
if (body.length > 0) {
|
||||
lines.push('```' + lang, body, '```', '');
|
||||
totalChars += body.length + opts.overhead;
|
||||
sourceSpent += body.length;
|
||||
newSourceChars += body.length;
|
||||
diag?.recordRender(filePath, opts.mode, body.length, opts.clipped || opts.covered.length > 0);
|
||||
if (opts.covered.length > 0) diag?.recordDedup(filePath, coveredChars(opts.covered), opts.covered);
|
||||
noteEmitted(filePath, [...ranges, ...opts.covered], body.length, fingerprint);
|
||||
renderedFilePaths.push(filePath);
|
||||
filesIncluded++;
|
||||
return;
|
||||
}
|
||||
// Fully held. The section is the pointer; the slot and the bytes go to a
|
||||
// file the agent has not seen. The spans are still recorded (at zero
|
||||
// bytes) because the record means "source the agent HAS", not "bytes
|
||||
// this call spent" — refreshing them keeps a long session from ageing
|
||||
// them out of the retained window and re-serving them for nothing.
|
||||
totalChars += opts.overhead;
|
||||
diag?.recordRender(filePath, 'backref', 0, false);
|
||||
diag?.recordDedup(filePath, coveredChars(opts.covered), opts.covered);
|
||||
noteEmitted(filePath, opts.covered, 0, fingerprint);
|
||||
renderedFilePaths.push(filePath);
|
||||
if (!suppressedFallback && opts.fullBody.length > 0) {
|
||||
suppressedFallback = {
|
||||
filePath,
|
||||
at,
|
||||
replacing: lines.length - at,
|
||||
section: [opts.header, '', '```' + lang, opts.fullBody, '```', ''],
|
||||
sourceChars: opts.fullBody.length,
|
||||
overhead: opts.overhead,
|
||||
ranges: opts.fullRanges,
|
||||
fingerprint,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
// Disk-drift gate (#1474): every render branch below except whole-file
|
||||
// slices fileContent (CURRENT bytes) at INDEXED line ranges. Content is
|
||||
@@ -4132,8 +4317,7 @@ export class ToolHandler {
|
||||
// Pass 2: render in line order — full body for chosen symbols, else the
|
||||
// signature line (capped, with a "+N more" tail so the structure map of a
|
||||
// god-file doesn't itself bloat the budget).
|
||||
const skel: string[] = [];
|
||||
const skelRanges: ExploreLineRange[] = [];
|
||||
const skel: Array<{ range: ExploreLineRange; text: string }> = [];
|
||||
let coveredUntil = 0; // skip symbols already inside an emitted body
|
||||
let sigCount = 0, sigDropped = 0;
|
||||
const SIG_MAX = Math.max(12, budget.maxSymbolsInFileHeader * 2);
|
||||
@@ -4142,8 +4326,10 @@ export class ToolHandler {
|
||||
if (bodyIds.has(n.id)) {
|
||||
const end = n.endLine;
|
||||
const body = fileLines.slice(n.startLine - 1, end).join('\n');
|
||||
skel.push(exploreLineNumbersEnabled() ? numberSourceLines(body, n.startLine) : body);
|
||||
skelRanges.push({ start: n.startLine, end });
|
||||
skel.push({
|
||||
range: { start: n.startLine, end },
|
||||
text: withLineNumbers ? numberSourceLines(body, n.startLine) : body,
|
||||
});
|
||||
coveredUntil = end;
|
||||
} else {
|
||||
// Elide the body, emit the signature. node.startLine can point at a
|
||||
@@ -4156,13 +4342,15 @@ export class ToolHandler {
|
||||
if (sigCount >= SIG_MAX) { sigDropped++; continue; }
|
||||
const sig = (fileLines[lineNo - 1] || '').trim();
|
||||
if (sig) {
|
||||
skel.push(exploreLineNumbersEnabled() ? `${lineNo}\t${sig}` : sig);
|
||||
skelRanges.push({ start: lineNo, end: lineNo });
|
||||
skel.push({
|
||||
range: { start: lineNo, end: lineNo },
|
||||
text: withLineNumbers ? `${lineNo}\t${sig}` : sig,
|
||||
});
|
||||
sigCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (sigDropped > 0) skel.push(`… +${sigDropped} more (signatures elided)`);
|
||||
const sigTail = sigDropped > 0 ? `… +${sigDropped} more (signatures elided)` : '';
|
||||
if (skel.length > 0) {
|
||||
const names = [...new Set(group.nodes.filter(n => n.kind !== 'import' && n.kind !== 'export').map(n => n.name))]
|
||||
.slice(0, budget.maxSymbolsInFileHeader).join(', ');
|
||||
@@ -4175,14 +4363,25 @@ export class ToolHandler {
|
||||
const tag = bodyIds.size > 0
|
||||
? 'focused (the methods you named in full, the rest as signatures — codegraph_explore a signature by name for its body; do NOT Read)'
|
||||
: 'skeleton (signatures only — codegraph_explore a name for its full body; do NOT Read)';
|
||||
lines.push(fileSectionHeader(filePath, `${names} · ${tag}`), '', '```' + lang, skel.join('\n'), '```', '');
|
||||
totalChars += skel.join('\n').length + 120;
|
||||
sourceSpent += skel.join('\n').length;
|
||||
// Always "clipped": the per-symbol view elides bodies by construction.
|
||||
diag?.recordRender(filePath, bodyIds.size > 0 ? 'focused' : 'skeleton', skel.join('\n').length, true);
|
||||
noteEmitted(filePath, skelRanges, skel.join('\n').length);
|
||||
renderedFilePaths.push(filePath);
|
||||
filesIncluded++;
|
||||
// Dedup runs on the per-symbol parts, so a body the agent already has
|
||||
// becomes a pointer while the signature map around it survives intact
|
||||
// (a one-line signature is far under MIN_COVERED_LINES and is never
|
||||
// withheld — the structure map is what makes this render legible).
|
||||
const dd = dedupeSpans(skel);
|
||||
const withTail = (parts: ReadonlyArray<{ text: string }>) =>
|
||||
[...parts.map((p) => p.text), ...(sigTail ? [sigTail] : [])].join('\n');
|
||||
emitFileSection({
|
||||
header: fileSectionHeader(filePath, `${names} · ${tag}`),
|
||||
body: dd.parts.length > 0 ? withTail(dd.parts) : '',
|
||||
ranges: dd.parts.map((p) => p.range),
|
||||
covered: dd.covered,
|
||||
overhead: 120,
|
||||
mode: bodyIds.size > 0 ? 'focused' : 'skeleton',
|
||||
// Always "clipped": the per-symbol view elides bodies by construction.
|
||||
clipped: true,
|
||||
fullBody: withTail(skel),
|
||||
fullRanges: skel.map((p) => p.range),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -4260,7 +4459,14 @@ export class ToolHandler {
|
||||
&& totalChars + fileContent.length + EXPLORE_ALLOCATION.FILE_OVERHEAD <= renderCeiling);
|
||||
if (fileLines.length <= WHOLE_FILE_MAX_LINES && buysWhole) {
|
||||
const body = fileContent.replace(/\n+$/, '');
|
||||
let wholeSection = exploreLineNumbersEnabled() ? numberSourceLines(body, 1) : body;
|
||||
const wholeRange: ExploreLineRange = { start: 1, end: body.split('\n').length };
|
||||
const fullSection = withLineNumbers ? numberSourceLines(body, 1) : body;
|
||||
// The buy decision above was made on the file's FULL size on purpose: it
|
||||
// asks "did this file's relevance earn all of itself", which dedup does
|
||||
// not change. Dedup then only ever makes the render smaller, so a buy
|
||||
// that was funded stays funded.
|
||||
const ddWhole = dedupeSpans([{ range: wholeRange, text: fullSection }]);
|
||||
const wholeSection = ddWhole.parts.map((p) => p.text).join(GAP_MARKER);
|
||||
const uniqSymbols = [...new Set(
|
||||
group.nodes
|
||||
.filter(n => n.kind !== 'import' && n.kind !== 'export')
|
||||
@@ -4281,14 +4487,18 @@ export class ToolHandler {
|
||||
diag?.recordSkip(filePath, 'budget-whole-file');
|
||||
continue;
|
||||
}
|
||||
lines.push(wholeHeader, '', '```' + lang, wholeSection, '```', '');
|
||||
totalChars += wholeSection.length + 200;
|
||||
sourceSpent += wholeSection.length;
|
||||
diag?.recordRender(filePath, 'whole', wholeSection.length, false);
|
||||
// The whole file, minus any trailing blank lines the render trimmed.
|
||||
noteEmitted(filePath, [{ start: 1, end: body.split('\n').length }], wholeSection.length);
|
||||
renderedFilePaths.push(filePath);
|
||||
filesIncluded++;
|
||||
emitFileSection({
|
||||
header: wholeHeader,
|
||||
body: wholeSection,
|
||||
// The whole file, minus any trailing blank lines the render trimmed.
|
||||
ranges: ddWhole.parts.map((p) => p.range),
|
||||
covered: ddWhole.covered,
|
||||
overhead: 200,
|
||||
mode: 'whole',
|
||||
clipped: false,
|
||||
fullBody: fullSection,
|
||||
fullRanges: [wholeRange],
|
||||
});
|
||||
if (fileStale) staleRendered.push(filePath);
|
||||
continue;
|
||||
}
|
||||
@@ -4441,10 +4651,6 @@ export class ToolHandler {
|
||||
// until the per-file char cap is hit. Truly enormous single clusters
|
||||
// get tail-trimmed with a marker.
|
||||
const contextPadding = 3;
|
||||
const withLineNumbers = exploreLineNumbersEnabled();
|
||||
// Language-neutral separator (no `//` — not a comment in Python, Ruby,
|
||||
// etc.). With line numbers on, the line-number jump also signals the gap.
|
||||
const GAP_MARKER = '\n\n... (gap) ...\n\n';
|
||||
// An oversize spine method (the call path runs THROUGH a god-method — n8n's
|
||||
// processRunExecutionData is 962 lines) is windowed to its next-hop CALL site
|
||||
// plus the signature head, NOT dumped whole. Without this the cluster is too big
|
||||
@@ -4453,42 +4659,50 @@ export class ToolHandler {
|
||||
// the spine's call still appears in context.
|
||||
const OVERSIZE_SPINE_LINES = 200;
|
||||
const SPINE_WINDOW = 28; // lines each side of the next-hop call site
|
||||
// Returns the rendered text AND the line spans it covers. The spans are
|
||||
// what the session record is built from (CG-17): reporting them from the
|
||||
// same function that slices the source is what keeps the record honest —
|
||||
// a second function mirroring these window/padding rules would drift, and
|
||||
// a record that claims lines it never sent withholds them from a later
|
||||
// call, which costs a Read.
|
||||
// Returns the rendered text as SPAN-KEYED PARTS. Every part carries the
|
||||
// exact line range its text was sliced from, which two things depend on:
|
||||
// the session record (CG-17) — a record claiming lines it never sent would
|
||||
// withhold them from a later call, costing a Read — and cross-call dedup
|
||||
// (CG-18), which rebuilds a part's text from a narrower span when the
|
||||
// agent already holds the rest. Both read the spans from the function that
|
||||
// does the slicing; a second function mirroring these window/padding rules
|
||||
// would drift.
|
||||
type SectionPart = { range: ExploreLineRange; text: string };
|
||||
const sectionText = (parts: ReadonlyArray<SectionPart>): string =>
|
||||
parts.map((p) => p.text).join(GAP_MARKER);
|
||||
const buildSection = (
|
||||
c: { start: number; end: number; hasSpine?: boolean; spineCallLine?: number },
|
||||
): { text: string; ranges: ExploreLineRange[] } => {
|
||||
): SectionPart[] => {
|
||||
if (c.hasSpine && c.spineCallLine && (c.end - c.start + 1) > OVERSIZE_SPINE_LINES) {
|
||||
const call = c.spineCallLine;
|
||||
const winStart = Math.max(c.start, call - SPINE_WINDOW);
|
||||
const winEnd = Math.min(c.end, call + SPINE_WINDOW);
|
||||
const parts: string[] = [];
|
||||
const spans: ExploreLineRange[] = [];
|
||||
const parts: SectionPart[] = [];
|
||||
// Signature head, only when it sits clearly above the window (else the
|
||||
// window already covers the method opening).
|
||||
const headEnd = Math.min(c.start + 4, winStart - 2);
|
||||
if (headEnd >= c.start) {
|
||||
const head = fileLines.slice(c.start - 1, headEnd).join('\n');
|
||||
parts.push(withLineNumbers ? numberSourceLines(head, c.start) : head);
|
||||
spans.push({ start: c.start, end: headEnd });
|
||||
parts.push({
|
||||
range: { start: c.start, end: headEnd },
|
||||
text: withLineNumbers ? numberSourceLines(head, c.start) : head,
|
||||
});
|
||||
}
|
||||
const win = fileLines.slice(winStart - 1, winEnd).join('\n');
|
||||
parts.push(withLineNumbers ? numberSourceLines(win, winStart) : win);
|
||||
spans.push({ start: winStart, end: winEnd });
|
||||
return { text: parts.join(GAP_MARKER), ranges: spans };
|
||||
parts.push({
|
||||
range: { start: winStart, end: winEnd },
|
||||
text: withLineNumbers ? numberSourceLines(win, winStart) : win,
|
||||
});
|
||||
return parts;
|
||||
}
|
||||
const startIdx = Math.max(0, c.start - 1 - contextPadding);
|
||||
const endIdx = Math.min(fileLines.length, c.end + contextPadding);
|
||||
const slice = fileLines.slice(startIdx, endIdx).join('\n');
|
||||
// startIdx is 0-based, so the slice's first line is line startIdx + 1.
|
||||
return {
|
||||
return [{
|
||||
range: { start: startIdx + 1, end: endIdx },
|
||||
text: withLineNumbers ? numberSourceLines(slice, startIdx + 1) : slice,
|
||||
ranges: [{ start: startIdx + 1, end: endIdx }],
|
||||
};
|
||||
}];
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -4507,10 +4721,7 @@ export class ToolHandler {
|
||||
* body is never cut, and the members are chosen by the same importance the
|
||||
* cluster ranking uses. Returns null when nothing needed shrinking.
|
||||
*/
|
||||
const shrinkCluster = (
|
||||
c: ExploreCluster,
|
||||
cap: number,
|
||||
): { text: string; ranges: ExploreLineRange[] } | null => {
|
||||
const shrinkCluster = (c: ExploreCluster, cap: number): SectionPart[] | null => {
|
||||
if (c.members.length < 2) return null;
|
||||
const byImportance = [...c.members].sort((a, b) =>
|
||||
b.importance - a.importance || (a.end - a.start) - (b.end - b.start) || a.start - b.start);
|
||||
@@ -4535,11 +4746,30 @@ export class ToolHandler {
|
||||
if (last && r.start <= last.end + gapThreshold) last.end = Math.max(last.end, r.end);
|
||||
else merged.push({ start: r.start, end: r.end });
|
||||
}
|
||||
const sections = merged.map((m) => buildSection(m));
|
||||
return {
|
||||
text: sections.map((s) => s.text).join(GAP_MARKER),
|
||||
ranges: sections.flatMap((s) => s.ranges),
|
||||
};
|
||||
return merged.flatMap((m) => buildSection(m));
|
||||
};
|
||||
|
||||
/**
|
||||
* One cluster's final parts: built, shrunk if it overruns `cap`, then
|
||||
* passed through the session history (CG-18).
|
||||
*
|
||||
* The shrink decision reads the DEDUPED length on purpose. A cluster whose
|
||||
* bytes the agent already holds costs this response nothing, so shrinking
|
||||
* it on its raw size would drop new symbols to make room for source that
|
||||
* is not being sent — spending the file's budget on nothing.
|
||||
*/
|
||||
const renderCluster = (
|
||||
c: ExploreCluster,
|
||||
cap: number,
|
||||
): { parts: SectionPart[]; covered: ExploreLineRange[]; shrunk: boolean } => {
|
||||
const base = dedupeSpans(buildSection(c));
|
||||
if (sectionText(base.parts).length <= cap) {
|
||||
return { parts: base.parts, covered: base.covered, shrunk: false };
|
||||
}
|
||||
const shrunk = shrinkCluster(c, cap);
|
||||
if (shrunk === null) return { parts: base.parts, covered: base.covered, shrunk: false };
|
||||
const dd = dedupeSpans(shrunk);
|
||||
return { parts: dd.parts, covered: dd.covered, shrunk: true };
|
||||
};
|
||||
|
||||
// Rank clusters for inclusion under the per-file cap. Entry-point
|
||||
@@ -4586,23 +4816,28 @@ export class ToolHandler {
|
||||
// spine can't run away or starve co-flow files entirely.
|
||||
const SPINE_CEILING = Math.min(Math.round(allowance * 1.5), headroom);
|
||||
const chosenIndices = new Set<number>();
|
||||
// Shrunk renders for oversize clusters, by cluster index (CG-12). Computed
|
||||
// Final renders (deduped, shrunk where oversize) by cluster index. Computed
|
||||
// during selection and reused at emission so the two never disagree.
|
||||
const shrunkSections = new Map<number, { text: string; ranges: ExploreLineRange[] }>();
|
||||
const renderedClusters = new Map<number, ReturnType<typeof renderCluster>>();
|
||||
let anyClusterShrunk = false;
|
||||
let projectedChars = 0;
|
||||
for (const rc of rankedClusters) {
|
||||
const sectionLen = buildSection(rc.c).text.length + (chosenIndices.size > 0 ? GAP_MARKER.length : 0);
|
||||
// The top-ranked cluster is always taken — an empty file section sends the
|
||||
// agent to Read, negating the savings. But "always taken" is not "taken at
|
||||
// any size": when it overruns the reservation it is SHRUNK to the
|
||||
// highest-importance whole symbol ranges inside it, so a single-cluster
|
||||
// god-file spends its allotment instead of the whole response's.
|
||||
if (chosenIndices.size === 0) {
|
||||
const cap = rc.c.hasSpine ? SPINE_CEILING : fileBudget;
|
||||
const shrunk = sectionLen > cap ? shrinkCluster(rc.c, cap) : null;
|
||||
if (shrunk !== null) shrunkSections.set(rc.idx, shrunk);
|
||||
// god-file spends its allotment instead of the whole response's. Later
|
||||
// clusters are never shrunk — they either fit or wait for another call.
|
||||
const first = chosenIndices.size === 0;
|
||||
const cap = rc.c.hasSpine ? SPINE_CEILING : fileBudget;
|
||||
const section = renderCluster(rc.c, first ? cap : Infinity);
|
||||
const text = sectionText(section.parts);
|
||||
const sectionLen = text.length + (!first && text.length > 0 ? GAP_MARKER.length : 0);
|
||||
if (first) {
|
||||
renderedClusters.set(rc.idx, section);
|
||||
anyClusterShrunk = anyClusterShrunk || section.shrunk;
|
||||
chosenIndices.add(rc.idx);
|
||||
projectedChars += shrunk !== null ? shrunk.text.length : sectionLen;
|
||||
projectedChars += sectionLen;
|
||||
continue;
|
||||
}
|
||||
// A spine cluster (the rendered call path) is the flow answer — include it
|
||||
@@ -4611,6 +4846,7 @@ export class ToolHandler {
|
||||
const fits = projectedChars + sectionLen <= fileBudget;
|
||||
const spineFits = rc.c.hasSpine && projectedChars + sectionLen <= SPINE_CEILING;
|
||||
if (!fits && !spineFits) continue;
|
||||
renderedClusters.set(rc.idx, section);
|
||||
chosenIndices.add(rc.idx);
|
||||
projectedChars += sectionLen;
|
||||
}
|
||||
@@ -4619,13 +4855,18 @@ export class ToolHandler {
|
||||
let fileSection = '';
|
||||
const allSymbols: string[] = [];
|
||||
const sectionRanges: ExploreLineRange[] = [];
|
||||
const coveredRanges: ExploreLineRange[] = [];
|
||||
for (let i = 0; i < clusters.length; i++) {
|
||||
if (!chosenIndices.has(i)) continue;
|
||||
const cluster = clusters[i]!;
|
||||
const section = shrunkSections.get(i) ?? buildSection(cluster);
|
||||
if (fileSection.length > 0) fileSection += GAP_MARKER;
|
||||
fileSection += section.text;
|
||||
sectionRanges.push(...section.ranges);
|
||||
const section = renderedClusters.get(i)!;
|
||||
const text = sectionText(section.parts);
|
||||
if (text.length > 0) {
|
||||
if (fileSection.length > 0) fileSection += GAP_MARKER;
|
||||
fileSection += text;
|
||||
}
|
||||
sectionRanges.push(...section.parts.map((p) => p.range));
|
||||
coveredRanges.push(...section.covered);
|
||||
allSymbols.push(...cluster.symbols);
|
||||
}
|
||||
|
||||
@@ -4635,7 +4876,7 @@ export class ToolHandler {
|
||||
// method is useless (the agent just Reads the rest for the other half), which
|
||||
// is the very fallback explore exists to prevent. A pathological file is
|
||||
// bounded by the cluster SELECTION above + the total hard ceiling.
|
||||
if (chosenIndices.size < clusters.length || shrunkSections.size > 0) {
|
||||
if (chosenIndices.size < clusters.length || anyClusterShrunk) {
|
||||
anyFileTrimmed = true;
|
||||
}
|
||||
|
||||
@@ -4673,19 +4914,62 @@ export class ToolHandler {
|
||||
continue;
|
||||
}
|
||||
|
||||
lines.push(fileHeader);
|
||||
lines.push('');
|
||||
lines.push('```' + lang);
|
||||
lines.push(fileSection);
|
||||
lines.push('```');
|
||||
lines.push('');
|
||||
// The undeduped render of the same clusters, needed only if this file ends
|
||||
// up fully back-referenced AND the whole call finds nothing new to say —
|
||||
// see `suppressedFallback`. Built lazily: on every other call it is dead
|
||||
// weight.
|
||||
const fullClusterParts = fileSection.length === 0
|
||||
? clusters.flatMap((c, i) => (chosenIndices.has(i) ? buildSection(c) : []))
|
||||
: [];
|
||||
emitFileSection({
|
||||
header: fileHeader,
|
||||
body: fileSection,
|
||||
ranges: sectionRanges,
|
||||
covered: mergeRanges(coveredRanges),
|
||||
overhead: 200,
|
||||
mode: 'clusters',
|
||||
clipped: chosenIndices.size < clusters.length,
|
||||
fullBody: sectionText(fullClusterParts),
|
||||
fullRanges: fullClusterParts.map((p) => p.range),
|
||||
});
|
||||
}
|
||||
|
||||
totalChars += fileSection.length + 200;
|
||||
sourceSpent += fileSection.length;
|
||||
diag?.recordRender(filePath, 'clusters', fileSection.length, chosenIndices.size < clusters.length);
|
||||
noteEmitted(filePath, sectionRanges, fileSection.length);
|
||||
renderedFilePaths.push(filePath);
|
||||
filesIncluded++;
|
||||
// Anti-abandonment restore (CG-18). Dedup withheld everything and nothing new
|
||||
// took its place — the response would be pointers only, which is the shape
|
||||
// that reads as "codegraph found nothing" and sends the agent to Read for
|
||||
// good. Put the top suppressed file back, in full, and keep its pointer off.
|
||||
// Deliberately checked against `newSourceChars` (source THIS call emitted)
|
||||
// rather than the response length: the flow and blast-radius sections are
|
||||
// always there, and they are not what makes a response feel sufficient.
|
||||
// Cast, not annotation: the only writer is the render loop's `emitFileSection`
|
||||
// closure, which TypeScript's flow analysis cannot see, so it narrows the
|
||||
// variable to `null` here and the truthiness check below would be `never`.
|
||||
const restore = suppressedFallback as SuppressedFallback | null;
|
||||
if (newSourceChars === 0 && restore) {
|
||||
if (totalChars + restore.sourceChars + restore.overhead <= renderCeiling) {
|
||||
lines.splice(restore.at, restore.replacing, ...restore.section);
|
||||
totalChars += restore.sourceChars + restore.overhead;
|
||||
sourceSpent += restore.sourceChars;
|
||||
newSourceChars += restore.sourceChars;
|
||||
filesIncluded++;
|
||||
const idx = backReferencedFiles.indexOf(restore.filePath);
|
||||
if (idx >= 0) backReferencedFiles.splice(idx, 1);
|
||||
emittedByFile.set(restore.filePath, {
|
||||
ranges: [...restore.ranges],
|
||||
bytes: restore.sourceChars,
|
||||
fingerprint: restore.fingerprint,
|
||||
});
|
||||
diag?.recordRender(restore.filePath, 'clusters', restore.sourceChars, false);
|
||||
diag?.recordDedup(restore.filePath, 0, []);
|
||||
}
|
||||
}
|
||||
|
||||
// The back-reference convention, stated once where the verbatim guarantee is
|
||||
// (#1474 does the same for drift). Without it a pointer reads as an
|
||||
// apology for missing source rather than as an index into source the agent
|
||||
// already has.
|
||||
if (backReferencedFiles.length > 0) {
|
||||
lines[verbatimHeaderIdx] += ` (Files marked **"Already sent earlier in this conversation"** are not repeated: their source came back on an earlier codegraph_explore call in THIS conversation and the file has not changed since, so that copy is exact and current — scroll back for it rather than re-fetching or Reading.)`;
|
||||
}
|
||||
|
||||
// Drift epilogue (#1474). The "verbatim / do not Read" guarantee above
|
||||
@@ -4833,12 +5117,22 @@ export class ToolHandler {
|
||||
// Session record (CG-17): only the files that SURVIVED the hard ceiling —
|
||||
// a section the truncation dropped was never delivered, and recording it
|
||||
// would let a later call withhold source the agent has never seen.
|
||||
//
|
||||
// A back-referenced file records its spans at ZERO bytes (CG-18) — it is
|
||||
// still source the agent holds for this file, which is what the record
|
||||
// means; dropping it would let the span age out of the retained window and
|
||||
// be re-served for nothing.
|
||||
const emittedFiles: ExploreFileEmission[] = [];
|
||||
let sourceBytes = 0;
|
||||
for (const fp of survivors) {
|
||||
const emitted = emittedByFile.get(fp);
|
||||
if (!emitted || emitted.bytes <= 0) continue;
|
||||
emittedFiles.push({ path: fp, ranges: emitted.ranges, bytes: emitted.bytes });
|
||||
if (!emitted || emitted.ranges.length === 0) continue;
|
||||
emittedFiles.push({
|
||||
path: fp,
|
||||
ranges: emitted.ranges,
|
||||
bytes: emitted.bytes,
|
||||
fingerprint: emitted.fingerprint,
|
||||
});
|
||||
sourceBytes += emitted.bytes;
|
||||
}
|
||||
return this.exploreResult(finalText, {
|
||||
|
||||
Reference in New Issue
Block a user