diff --git a/__tests__/explore-cross-call-dedup.test.ts b/__tests__/explore-cross-call-dedup.test.ts new file mode 100644 index 0000000..608d85c --- /dev/null +++ b/__tests__/explore-cross-call-dedup.test.ts @@ -0,0 +1,360 @@ +/** + * Cross-call source dedup (CG-18). + * + * A later `codegraph_explore` call in a session must not re-send source an + * earlier call already delivered — but every byte it withholds has to be + * replaced by a POINTER, never a silence. That asymmetry is what this suite + * guards, because the two failure directions cost wildly different amounts: a + * duplicate range wastes a few thousand chars, while a response that reads as + * "codegraph doesn't have it" costs a Read — and one or two of those early in a + * session teach an agent to stop calling the tool at all. + * + * Three layers: + * 1. the range algebra — what is withheld, and the thresholds that stop it + * from shredding a block into slivers; + * 2. the fingerprint gate — an edit between two calls must re-emit, since a + * pointer to pre-edit source is worse than no dedup at all; + * 3. the handler seam — a real second call against a real index: no duplicate + * ranges, a pointer for everything withheld, the reclaimed budget spent on + * source the agent has NOT seen, and never an all-pointer response. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; +import { ExploreSessionState, type ExploreProjectState } from '../src/mcp/explore-session-state'; +import { + EXPLORE_DEDUP, + dedupeRange, + fileFingerprint, + formatBackReference, + intersectRange, + mergeRanges, + servedRangesForFile, + subtractRange, + symbolsInSpans, +} from '../src/mcp/explore-dedup'; + +const FIXTURE_SRC = path.join(__dirname, 'fixtures', 'payroll-go'); +const QUERY = 'how does payroll cycle create and calculate payslips?'; +const POINTER = 'Already sent earlier in this conversation'; + +/** A prior-state shaped like the session tracker's, for the algebra tests. */ +function prior(files: Array<{ path: string; ranges: Array<[number, number]>; fingerprint?: string }>): ExploreProjectState { + return { + projectRoot: '/repo', + callCount: 1, + responseBytes: 1000, + calls: [{ + index: 1, + projectRoot: '/repo', + query: 'q', + sourceBytes: 500, + responseBytes: 1000, + files: files.map((f) => ({ + path: f.path, + ranges: f.ranges.map(([start, end]) => ({ start, end })), + bytes: 500, + fingerprint: f.fingerprint, + })), + }], + }; +} + +describe('range algebra', () => { + it('subtracts a held span out of the middle of an intended one', () => { + expect(subtractRange({ start: 1, end: 100 }, [{ start: 20, end: 40 }])) + .toEqual([{ start: 1, end: 19 }, { start: 41, end: 100 }]); + }); + + it('subtracts held spans at either edge, and a full cover to nothing', () => { + expect(subtractRange({ start: 10, end: 50 }, [{ start: 1, end: 20 }])) + .toEqual([{ start: 21, end: 50 }]); + expect(subtractRange({ start: 10, end: 50 }, [{ start: 30, end: 90 }])) + .toEqual([{ start: 10, end: 29 }]); + expect(subtractRange({ start: 10, end: 50 }, [{ start: 1, end: 90 }])).toEqual([]); + }); + + it('intersects to exactly what both sides hold', () => { + expect(intersectRange({ start: 10, end: 50 }, [{ start: 1, end: 20 }, { start: 45, end: 80 }])) + .toEqual([{ start: 10, end: 20 }, { start: 45, end: 50 }]); + expect(intersectRange({ start: 10, end: 50 }, [{ start: 60, end: 80 }])).toEqual([]); + }); + + it('merges touching spans — two adjacent blocks are one block of source', () => { + expect(mergeRanges([{ start: 5, end: 9 }, { start: 10, end: 12 }, { start: 40, end: 41 }])) + .toEqual([{ start: 5, end: 12 }, { start: 40, end: 41 }]); + }); + + it('emits ONLY the delta when a later call wants a wider window', () => { + // Call 1 sent the method; call 2 wants the class around it. + const { emit, covered } = dedupeRange({ start: 80, end: 200 }, [{ start: 100, end: 140 }]); + expect(covered).toEqual([{ start: 100, end: 140 }]); + expect(emit).toEqual([{ start: 80, end: 99 }, { start: 141, end: 200 }]); + }); + + it('withholds nothing when the overlap is smaller than a chunk worth pointing at', () => { + // Context padding and signature lines land here. Replacing them costs more + // in pointer text than the source is worth, and shreds the block. + const overlap = EXPLORE_DEDUP.MIN_COVERED_LINES - 1; + const { emit, covered } = dedupeRange({ start: 1, end: 100 }, [{ start: 10, end: 10 + overlap - 1 }]); + expect(covered).toEqual([]); + expect(emit).toEqual([{ start: 1, end: 100 }]); + }); + + it('leaves an untouched span exactly as it was', () => { + expect(dedupeRange({ start: 1, end: 50 }, [{ start: 200, end: 400 }])) + .toEqual({ emit: [{ start: 1, end: 50 }], covered: [] }); + expect(dedupeRange({ start: 1, end: 50 }, [])) + .toEqual({ emit: [{ start: 1, end: 50 }], covered: [] }); + }); +}); + +describe('the fingerprint gate', () => { + const FP = fileFingerprint('package main\nfunc main() {}\n'); + + it('returns the spans a call served for a file whose bytes are unchanged', () => { + const state = prior([{ path: 'a.go', ranges: [[1, 40], [60, 80]], fingerprint: FP }]); + expect(servedRangesForFile(state, 'a.go', FP)).toEqual([{ start: 1, end: 40 }, { start: 60, end: 80 }]); + }); + + it('returns NOTHING once the file has been edited — a pointer would be wrong', () => { + const state = prior([{ path: 'a.go', ranges: [[1, 40]], fingerprint: FP }]); + const edited = fileFingerprint('package main\nfunc main() { changed() }\n'); + expect(servedRangesForFile(state, 'a.go', edited)).toEqual([]); + }); + + it('ignores a record that cannot prove what it served', () => { + const state = prior([{ path: 'a.go', ranges: [[1, 40]] }]); + expect(servedRangesForFile(state, 'a.go', FP)).toEqual([]); + }); + + it('never crosses files, and is empty for an untracked session', () => { + const state = prior([{ path: 'a.go', ranges: [[1, 40]], fingerprint: FP }]); + expect(servedRangesForFile(state, 'b.go', FP)).toEqual([]); + expect(servedRangesForFile(null, 'a.go', FP)).toEqual([]); + }); + + it('distinguishes two files that hash the same prefix but differ in length', () => { + expect(fileFingerprint('abc')).not.toBe(fileFingerprint('abcd')); + expect(fileFingerprint('abc')).toBe(fileFingerprint('abc')); + }); +}); + +describe('the back-reference itself', () => { + const covered = [{ start: 100, end: 240 }]; + + it('names the file, the span and the symbols, and says the copy is still good', () => { + const text = formatBackReference('internal/x.go', covered, ['RunCycle', 'BuildPayslip'], { partial: false }); + expect(text).toContain('internal/x.go'); + expect(text).toContain('L100-240'); + expect(text).toContain('RunCycle, BuildPayslip'); + expect(text).toContain(POINTER); + expect(text).toContain('unchanged on disk'); + }); + + it('never tells the agent to Read, in either shape', () => { + for (const partial of [true, false]) { + const text = formatBackReference('x.go', covered, ['A'], { partial }); + expect(text).toMatch(/do NOT Read/i); + expect(text).not.toMatch(/\bRead (this|the) file (for|to)\b/i); + expect(text).not.toMatch(/omitted|unavailable|could not/i); + } + }); + + it('says the block below is only the NEW lines when the call still sends some', () => { + expect(formatBackReference('x.go', covered, [], { partial: true })).toContain('NEW lines'); + expect(formatBackReference('x.go', covered, [], { partial: false })).toContain('not repeated here'); + }); + + it('names only symbols that actually fall in the withheld spans', () => { + const nodes = [ + { name: 'InSpan', kind: 'function', startLine: 110, endLine: 130 }, + { name: 'Outside', kind: 'function', startLine: 300, endLine: 320 }, + { name: 'AnImport', kind: 'import', startLine: 105, endLine: 105 }, + ]; + expect(symbolsInSpans(nodes, covered)).toEqual(['InSpan']); + }); +}); + +describe('a second call against a real index', () => { + let testDir: string; + let cg: CodeGraph; + let handler: ToolHandler; + + beforeAll(async () => { + testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cg18-')); + fs.cpSync(FIXTURE_SRC, testDir, { recursive: true }); + fs.rmSync(path.join(testDir, '.codegraph'), { recursive: true, force: true }); + cg = CodeGraph.initSync(testDir); + await cg.indexAll(); + handler = new ToolHandler(cg); + }, 120_000); + + afterAll(() => { + if (cg) cg.destroy(); + if (testDir && fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true, force: true }); + }); + + const explore = (query: string, session?: ExploreSessionState, args: Record = {}) => + handler.execute('codegraph_explore', { query, ...args }, session).then((r) => r.content[0]!.text); + + /** + * The line numbers actually inside each file's fenced source. Read off the + * RESPONSE, not the bookkeeping — "no duplicate ranges" is a claim about what + * the agent received, and checking it against the record we also wrote would + * prove only that the two agree. + */ + function fencedLines(text: string): Map> { + const out = new Map>(); + let current: string | null = null; + let inFence = false; + for (const line of text.split('\n')) { + const header = /^\*\*`([^`]+)`\*\*/.exec(line); + if (header && !inFence) { current = header[1]!; continue; } + if (!inFence && current && line.startsWith('```')) { inFence = true; continue; } + if (inFence && line === '```') { inFence = false; continue; } + if (!inFence || !current) continue; + const numbered = /^(\d+)\t/.exec(line); + if (!numbered) continue; + if (!out.has(current)) out.set(current, new Set()); + out.get(current)!.add(Number(numbered[1])); + } + return out; + } + + it('never re-sends a line it already sent, and points at every line it withholds', async () => { + const session = new ExploreSessionState(); + const first = await explore(QUERY, session); + const second = await explore(QUERY, session); + + const before = fencedLines(first); + const after = fencedLines(second); + expect(after.size).toBeGreaterThan(0); + + // Every file whose source the second call withheld carries a pointer, and + // the pointer names it. + expect(second).toContain(POINTER); + for (const [file, lines] of before) { + const repeated = [...(after.get(file) ?? [])].filter((n) => lines.has(n)); + if (repeated.length === 0) continue; + // The only sanctioned repeat is the anti-abandonment restore, which fires + // ONLY when the call found nothing new to say — and this one did. + throw new Error(`call 2 re-sent ${file} lines ${repeated.slice(0, 5).join(',')}`); + } + }, 120_000); + + it('spends the reclaimed bytes on source the agent has not seen', async () => { + const session = new ExploreSessionState(); + const first = await explore(QUERY, session); + const second = await explore(QUERY, session); + + const before = fencedLines(first); + const after = fencedLines(second); + const fresh = [...after.entries()].reduce( + (sum, [file, lines]) => sum + [...lines].filter((n) => !(before.get(file)?.has(n))).length, 0); + // Not merely "smaller": a shrunken response is what dedup must NOT produce. + // The freed budget has to come back as lines the first call never sent. + expect(fresh).toBeGreaterThan(20); + expect(second.length).toBeLessThan(first.length); + }, 120_000); + + it('re-emits in full when the file changed between the two calls', async () => { + const session = new ExploreSessionState(); + const target = path.join(testDir, 'internal/usecase/payroll/payslip_builder.go'); + const original = fs.readFileSync(target, 'utf-8'); + try { + const first = await explore(QUERY, session); + expect(fencedLines(first).has('internal/usecase/payroll/payslip_builder.go')).toBe(true); + + fs.writeFileSync(target, original.replace('func sumKind(', 'func sumKindRenamed('), 'utf-8'); + const second = await explore(QUERY, session); + + // The edited file is served again, whole — a pointer here would send the + // agent to a copy of the file that no longer exists. + const pointerLines = second.split('\n').filter((l) => l.includes(POINTER)); + expect(pointerLines.some((l) => l.includes('payslip_builder.go'))).toBe(false); + expect(fencedLines(second).get('internal/usecase/payroll/payslip_builder.go')?.size ?? 0) + .toBeGreaterThan(20); + } finally { + fs.writeFileSync(target, original, 'utf-8'); + } + }, 120_000); + + it('always returns real source, even when the session already holds everything', async () => { + const session = new ExploreSessionState(); + await explore(QUERY, session); + await explore(QUERY, session); + const third = await explore(QUERY, session); + const fourth = await explore(QUERY, session); + + // An all-pointer response is the shape that reads as failure. Every call + // keeps at least one real fenced block, however much the session holds. + for (const [n, text] of [[3, third], [4, fourth]] as const) { + const lines = [...fencedLines(text).values()].reduce((s, set) => s + set.size, 0); + expect(lines, `call ${n} returned no source at all`).toBeGreaterThan(10); + } + }, 180_000); + + it('leaves the first call of a session untouched', async () => { + const tracked = await explore(QUERY, new ExploreSessionState()); + const untracked = await explore(QUERY); + expect(tracked).toBe(untracked); + }, 120_000); + + it('keeps two sessions on one handler independent', async () => { + const a = new ExploreSessionState(); + const b = new ExploreSessionState(); + const firstForA = await explore(QUERY, a); + await explore(QUERY, a); + // B's first call has seen nothing, whatever A has been served. + expect(await explore(QUERY, b)).toBe(firstForA); + }, 180_000); + + it('is off entirely under CODEGRAPH_EXPLORE_DEDUP=0', async () => { + const session = new ExploreSessionState(); + const previous = process.env.CODEGRAPH_EXPLORE_DEDUP; + process.env.CODEGRAPH_EXPLORE_DEDUP = '0'; + try { + const first = await explore(QUERY, session); + const second = await explore(QUERY, session); + expect(second).toBe(first); + expect(second).not.toContain(POINTER); + } finally { + if (previous === undefined) delete process.env.CODEGRAPH_EXPLORE_DEDUP; + else process.env.CODEGRAPH_EXPLORE_DEDUP = previous; + } + }, 120_000); + + it('reports the reclaimed bytes through the CG-4 diagnostic', async () => { + const sidecar = path.join(testDir, 'cg18-diagnostic.jsonl'); + const session = new ExploreSessionState(); + const previous = process.env.CODEGRAPH_EXPLORE_DEBUG; + process.env.CODEGRAPH_EXPLORE_DEBUG = sidecar; + try { + await explore(QUERY, session); + await explore(QUERY, session); + } finally { + if (previous === undefined) delete process.env.CODEGRAPH_EXPLORE_DEBUG; + else process.env.CODEGRAPH_EXPLORE_DEBUG = previous; + } + const [one, two] = fs.readFileSync(sidecar, 'utf-8').trim().split('\n').map((l) => JSON.parse(l)); + + expect(one.dedup.savedChars).toBe(0); + expect(two.dedup.savedChars).toBeGreaterThan(1000); + expect(two.dedup.backReferenced.length).toBeGreaterThan(0); + + // The reclamation is legible file by file: a back-referenced file spent + // none of its reservation, and the response still filled its envelope. + const backref = two.files.filter((f: { render: string }) => f.render === 'backref'); + for (const f of backref) { + expect(f.emittedChars).toBe(0); + expect(f.dedupSavedChars).toBeGreaterThan(0); + expect(f.dedupCovered.length).toBeGreaterThan(0); + } + const spentOnFreshSource = two.files.reduce((s: number, f: { emittedChars: number }) => s + f.emittedChars, 0); + expect(spentOnFreshSource).toBeGreaterThan(0); + }, 120_000); +}); diff --git a/src/mcp/explore-dedup.ts b/src/mcp/explore-dedup.ts new file mode 100644 index 0000000..541945c --- /dev/null +++ b/src/mcp/explore-dedup.ts @@ -0,0 +1,249 @@ +/** + * Cross-call source dedup for `codegraph_explore` (CG-18). + * + * The session record (CG-17) knows what earlier calls already sent. This module + * is the algebra that turns that record into a decision for the call being + * rendered: of the line ranges this call WOULD emit, which does the agent + * already hold, and what is genuinely new. + * + * Three rules shape everything here, and all three come from the same place — + * an insufficient-feeling response is what sends an agent to Read, and one or + * two of those early in a session teach it to abandon codegraph entirely + * (CLAUDE.md): + * + * 1. **A pointer, never a bare omission.** Removed source is replaced by a + * back-reference naming the file, the symbols, and the line span, worded so + * it is unmistakable that the source was already delivered IN THIS + * CONVERSATION and is still current. Silence reads as "codegraph didn't + * find it". + * 2. **Only prove-it dedup.** A span is withheld only when the file's bytes + * are byte-identical to what was served (a content fingerprint, not an + * mtime and not the index's drift flag). An edit between calls means the + * agent's copy is wrong, so the source is re-emitted in full. + * 3. **Cut chunks, not slivers.** Only a covered run of at least + * {@link EXPLORE_DEDUP.MIN_COVERED_LINES} lines is worth replacing. Below + * that the pointer costs more than the source, and shattering a block into + * one-line fragments produces exactly the ragged output that reads as a + * failure. Everything not withheld is emitted — where the algebra is + * unsure, it re-serves. + * + * Which way to be wrong, restated for this layer: re-serving something the agent + * has is a few hundred wasted chars; withholding something it never saw is a + * Read. Every threshold below leans to the first. + */ + +import { createHash } from 'crypto'; +import type { ExploreLineRange, ExploreProjectState } from './explore-session-state'; + +export const EXPLORE_DEDUP = { + /** + * Shortest already-served run that may be replaced by a back-reference. + * + * Sized against what dedup is actually FOR — a later call re-serving a whole + * method or file it already sent. A shorter covered run is either a signature + * line in a skeleton render or the ±3 lines of context padding around a + * cluster, and swapping either for a pointer trades bytes for noise: the + * pointer sentence is itself ~140 chars, so under this length dedup would + * make the response BIGGER while making it read as full of holes. + */ + MIN_COVERED_LINES: 8, + /** + * Below this many chars of NEW source, a file's remainder is folded into its + * back-reference instead of being fenced on its own. + * + * The shape this exists for, seen on the CG-17 fixture: a third call whose + * only unheld line was the file's trailing blank one, rendered as a code fence + * containing `228\t`. A fence holding two lines of nothing reads as a broken + * response, and reading as broken is the expensive failure — it is the thing + * that sends an agent to Read and keeps it there. So a remainder this small is + * dropped rather than shown. It is the one place this module withholds + * something the agent has not seen, and it is bounded to ~two lines that sit + * directly against source the agent does hold; the file is still named, with + * its symbols, so one follow-up explore fetches it whole. + */ + MIN_DELTA_CHARS: 160, + /** Line spans named in one pointer before it summarises the rest. */ + MAX_SPANS_IN_POINTER: 4, + /** Symbols named in one pointer before it summarises the rest. */ + MAX_SYMBOLS_IN_POINTER: 5, +} as const; + +const OFF = new Set(['0', 'false', 'off', 'no']); + +/** + * Kill switch: `CODEGRAPH_EXPLORE_DEDUP=0` renders every call as if the session + * had no history. Read per call (not memoized) so a test can toggle it. + */ +export function exploreDedupEnabled(): boolean { + const raw = process.env.CODEGRAPH_EXPLORE_DEDUP; + if (raw === undefined) return true; + return !OFF.has(raw.trim().toLowerCase()); +} + +/** + * Identity of the bytes a call served for one file. + * + * This — not the index's drift flag — is what gates dedup. `isFileStaleOnDisk` + * answers "did the file change since the last INDEX SYNC", which is a different + * question with a different answer: two calls inside one drift window served the + * same current bytes (dedup is correct), while a file edited and re-synced + * between two calls is never "stale" and yet the agent's copy is now wrong + * (dedup would be actively harmful). Length is prefixed so a hash prefix + * collision cannot alias two files of different size. + */ +export function fileFingerprint(content: string): string { + return `${content.length}:${createHash('sha1').update(content).digest('hex').slice(0, 16)}`; +} + +/** Sort + merge overlapping/adjacent spans into the smallest equivalent set. */ +export function mergeRanges(ranges: ReadonlyArray): ExploreLineRange[] { + const valid = ranges + .filter((r) => Number.isFinite(r.start) && Number.isFinite(r.end) && r.end >= r.start && r.start >= 1) + .map((r) => ({ start: Math.floor(r.start), end: Math.floor(r.end) })) + .sort((a, b) => a.start - b.start || a.end - b.end); + const out: ExploreLineRange[] = []; + for (const r of valid) { + const last = out[out.length - 1]; + if (last && r.start <= last.end + 1) last.end = Math.max(last.end, r.end); + else out.push({ ...r }); + } + return out; +} + +/** The parts of `range` that `served` covers. */ +export function intersectRange( + range: ExploreLineRange, + served: ReadonlyArray, +): ExploreLineRange[] { + const out: ExploreLineRange[] = []; + for (const s of served) { + const start = Math.max(range.start, s.start); + const end = Math.min(range.end, s.end); + if (end >= start) out.push({ start, end }); + } + return mergeRanges(out); +} + +/** The parts of `range` that `cut` does NOT cover. */ +export function subtractRange( + range: ExploreLineRange, + cut: ReadonlyArray, +): ExploreLineRange[] { + const out: ExploreLineRange[] = []; + let cursor = range.start; + for (const c of mergeRanges(cut)) { + if (c.end < cursor) continue; + if (c.start > range.end) break; + if (c.start > cursor) out.push({ start: cursor, end: Math.min(c.start - 1, range.end) }); + cursor = Math.max(cursor, c.end + 1); + if (cursor > range.end) break; + } + if (cursor <= range.end) out.push({ start: cursor, end: range.end }); + return out; +} + +/** What one intended span becomes once the session's history is applied. */ +export interface RangeDedup { + /** Spans to render now — everything not proven-already-held. */ + emit: ExploreLineRange[]; + /** Spans replaced by a back-reference. */ + covered: ExploreLineRange[]; +} + +/** + * Split one intended span into what to emit and what to point back at. + * + * Covered runs shorter than {@link EXPLORE_DEDUP.MIN_COVERED_LINES} are left in + * the emit set on purpose (rule 3 above) — so a span the agent holds "almost + * all of" still comes back whole rather than as a stutter of fragments around + * pointers. + */ +export function dedupeRange( + range: ExploreLineRange, + served: ReadonlyArray, + minCovered: number = EXPLORE_DEDUP.MIN_COVERED_LINES, +): RangeDedup { + if (served.length === 0 || range.end < range.start) return { emit: [range], covered: [] }; + const covered = intersectRange(range, served).filter((r) => r.end - r.start + 1 >= minCovered); + if (covered.length === 0) return { emit: [range], covered: [] }; + return { emit: subtractRange(range, covered), covered }; +} + +/** + * Every line span this session has already served for one file, but ONLY from + * calls that served the SAME BYTES. + * + * A record with no fingerprint is ignored rather than trusted: it cannot prove + * the agent's copy matches the file on disk now, and an unprovable match is + * exactly the case where re-serving is right. + */ +export function servedRangesForFile( + prior: ExploreProjectState | null, + filePath: string, + fingerprint: string, +): ExploreLineRange[] { + if (!prior) return []; + const spans: ExploreLineRange[] = []; + for (const call of prior.calls) { + for (const file of call.files) { + if (file.path !== filePath) continue; + if (!file.fingerprint || file.fingerprint !== fingerprint) continue; + spans.push(...file.ranges); + } + } + return mergeRanges(spans); +} + +/** `L12`, `L12-40`, capped with a `+N more` tail. */ +export function formatSpans(spans: ReadonlyArray): string { + const shown = spans.slice(0, EXPLORE_DEDUP.MAX_SPANS_IN_POINTER) + .map((r) => (r.start === r.end ? `L${r.start}` : `L${r.start}-${r.end}`)) + .join(', '); + const more = spans.length - EXPLORE_DEDUP.MAX_SPANS_IN_POINTER; + return more > 0 ? `${shown}, +${more} more span${more === 1 ? '' : 's'}` : shown; +} + +/** + * The line that replaces withheld source. + * + * It has one job: make the agent reach into its own context instead of into + * Read. So it carries the three things needed to find the source it already has + * — path, symbols, line spans — plus the two facts that make using it safe: + * that it came from THIS conversation, and that the file has not changed since + * (which is checked, not asserted — see {@link fileFingerprint}). It never says + * "omitted", and it never steers to Read. + */ +export function formatBackReference( + filePath: string, + covered: ReadonlyArray, + symbols: ReadonlyArray, + opts: { partial: boolean }, +): string { + const names = symbols.slice(0, EXPLORE_DEDUP.MAX_SYMBOLS_IN_POINTER); + const moreNames = symbols.length - names.length; + const symbolPart = names.length > 0 + ? ` (${names.join(', ')}${moreNames > 0 ? `, +${moreNames} more` : ''})` + : ''; + const head = `> **Already sent earlier in this conversation:** \`${filePath}\` ${formatSpans(covered)}${symbolPart}`; + const tail = opts.partial + ? ' — unchanged on disk since, so that copy is still exact. Only the NEW lines are shown below; scroll back for the rest. Do NOT Read this file.' + : ' — unchanged on disk since, so that copy is still exact and is not repeated here. Use it from your context; do NOT Read this file.'; + return head + tail; +} + +/** Symbol names whose definitions fall inside the withheld spans. */ +export function symbolsInSpans( + nodes: ReadonlyArray<{ name: string; kind: string; startLine: number; endLine: number }>, + spans: ReadonlyArray, +): string[] { + const out: string[] = []; + const seen = new Set(); + for (const n of nodes) { + if (n.kind === 'import' || n.kind === 'export') continue; + if (!spans.some((s) => n.startLine <= s.end && (n.endLine || n.startLine) >= s.start)) continue; + if (seen.has(n.name)) continue; + seen.add(n.name); + out.push(n.name); + } + return out; +} diff --git a/src/mcp/explore-diagnostics.ts b/src/mcp/explore-diagnostics.ts index c54273a..941fd4c 100644 --- a/src/mcp/explore-diagnostics.ts +++ b/src/mcp/explore-diagnostics.ts @@ -41,6 +41,7 @@ export type ExploreRenderMode = | 'focused' // per-symbol view, named/spine bodies full | 'skeleton' // per-symbol view, signatures only | 'stale-omitted' // drifted on disk; source deliberately withheld + | 'backref' // fully served by an earlier call this session (CG-18) | 'dropped'; // rendered into `lines` but cut by the final hard ceiling /** Why a ranked candidate never reached the output. */ @@ -92,6 +93,17 @@ interface FileRecord extends ExploreCandidateMeta { */ allowance: number | null; render?: ExploreRenderMode; + /** + * Source chars this call did NOT re-send because an earlier call in the + * session already did (CG-18). Reclaimed, not lost: it leaves through + * `sourceSpent` (the carry-forward pool hands it to lower-ranked files) and, + * for a fully back-referenced file, through the freed `maxFiles` slot. The + * reallocation is legible as the difference between this file's + * `allowance` and `emittedChars` against the files below it in the table. + */ + dedupSavedChars: number; + /** Line spans replaced by a back-reference. */ + dedupCovered: Array<[number, number]>; /** Source chars the render loop handed to `lines` (pre-final-truncation). */ emittedChars: number; /** Source chars present in the FINAL text — authoritative, truncation-aware. */ @@ -130,6 +142,8 @@ export interface ExploreDiagnosticFile extends ExploreCandidateMeta { render: ExploreRenderMode | null; skipped: ExploreSkipReason | null; clipped: boolean; + dedupSavedChars: number; + dedupCovered: Array<[number, number]>; emittedChars: number; finalChars: number; share: number; @@ -191,6 +205,17 @@ export interface ExploreDiagnosticReport { filesRenderedByLoop: number; filesInFinalOutput: number; }; + /** + * Cross-call source dedup (CG-18): what this call did NOT re-send because an + * earlier call in this session already sent it, and where those bytes went. + * `savedChars` 0 with a non-empty session block means nothing overlapped. + */ + dedup: { + savedChars: number; + backReferenced: string[]; + /** Files fully replaced by a pointer — each one also freed a `maxFiles` slot. */ + fullyBackReferenced: string[]; + }; /** The proportional split (CG-12): what each file was promised, and why. */ allocation: { /** Chars divided among admitted files (envelope minus per-file overhead). */ @@ -338,6 +363,7 @@ export class ExploreDiagnostics { noteCandidate(path: string, meta: ExploreCandidateMeta): void { this.files.set(path, { path, ...meta, allowance: null, + dedupSavedChars: 0, dedupCovered: [], emittedChars: 0, finalChars: 0, share: 0, allocatedShare: 0, clipped: false, }); } @@ -375,6 +401,19 @@ export class ExploreDiagnostics { rec.skipped = undefined; } + /** + * Source this call withheld because the session already holds it (CG-18). + * Called with `(path, 0, [])` to clear a record — the anti-abandonment restore + * puts a suppressed file's source back, and a diagnostic still claiming the + * saving would misreport where the envelope went. + */ + recordDedup(path: string, savedChars: number, covered: ReadonlyArray<{ start: number; end: number }>): void { + const rec = this.files.get(path); + if (!rec) return; + rec.dedupSavedChars = savedChars; + rec.dedupCovered = covered.map((r) => [r.start, r.end] as [number, number]); + } + /** * A candidate was passed over before rendering. First reason wins — the * blanket `max-files` sweep must not overwrite a file's specific reason. @@ -410,8 +449,10 @@ export class ExploreDiagnostics { rec.share = envelope > 0 ? rec.finalChars / envelope : 0; rec.allocatedShare = allocatedChars > 0 ? rec.emittedChars / allocatedChars : 0; // Rendered into `lines` but absent from the final text → the hard - // ceiling dropped its whole section. - if (rec.render && rec.render !== 'stale-omitted' && rec.finalChars === 0) { + // ceiling dropped its whole section. A back-referenced file has no + // fenced source BY DESIGN (CG-18), so it is never "dropped". + if (rec.render && rec.render !== 'stale-omitted' && rec.render !== 'backref' + && rec.finalChars === 0) { rec.render = 'dropped'; rec.clipped = true; } @@ -467,6 +508,11 @@ export class ExploreDiagnostics { filesRenderedByLoop: filesIncluded, filesInFinalOutput: rendered.length, }, + dedup: { + savedChars: records.reduce((s, r) => s + r.dedupSavedChars, 0), + backReferenced: records.filter((r) => r.dedupSavedChars > 0).map((r) => r.path), + fullyBackReferenced: records.filter((r) => r.render === 'backref').map((r) => r.path), + }, allocation: { pool: this.allocPool, cliffAt: round6(this.allocCliffAt), @@ -495,6 +541,8 @@ export class ExploreDiagnostics { render: r.render ?? null, skipped: r.skipped ?? null, clipped: r.clipped, + dedupSavedChars: r.dedupSavedChars, + dedupCovered: r.dedupCovered.map((s) => [...s] as [number, number]), emittedChars: r.emittedChars, finalChars: r.finalChars, share: round6(r.share), @@ -614,6 +662,15 @@ export function renderTable(report: ExploreDiagnosticReport): string { ` relevance gate ${sel.graphGateApplied ? 'applied' : 'not applied'}` + ` at graph >= ${sel.graphGateThreshold.toFixed(5)} (6% of max ${sel.maxGraph.toFixed(5)})`, ); + const dedup = report.dedup; + if (dedup && dedup.savedChars > 0) { + out.push( + ` dedup ${num(dedup.savedChars)} chars not re-sent` + + ` · ${dedup.fullyBackReferenced.length} file(s) fully back-referenced` + + ` (each also freed a maxFiles slot)` + + (dedup.backReferenced.length > 0 ? `: ${dedup.backReferenced.join(', ')}` : ''), + ); + } const alloc = report.allocation; out.push( ` allocation ${num(alloc.reserved)} reserved of ${num(alloc.pool)} pool` + @@ -627,7 +684,7 @@ export function renderTable(report: ExploreDiagnosticReport): string { // Allocated (not delivered) is the allocator's own decision — the number the // budget work is about. Delivered is what the agent got. They differ only // when the ceiling truncated; showing both makes that divergence obvious. - const shown = files.filter((f) => f.emittedChars > 0 || f.finalChars > 0); + const shown = files.filter((f) => f.emittedChars > 0 || f.finalChars > 0 || f.render === 'backref'); if (shown.length > 0) { out.push(' # alloc% deliv% bytes reserved score graph hits pen flags render file'); for (const f of shown) { @@ -647,6 +704,11 @@ export function renderTable(report: ExploreDiagnosticReport): string { f.path, ); out.push(' kinds: ' + (f.kinds || '-')); + if (f.dedupSavedChars > 0) { + const spans = f.dedupCovered.slice(0, 6).map(([a, b]) => (a === b ? `${a}` : `${a}-${b}`)).join(','); + const more = f.dedupCovered.length > 6 ? `,+${f.dedupCovered.length - 6}` : ''; + out.push(` dedup: ${num(f.dedupSavedChars)} chars already sent this session · L${spans}${more}`); + } } out.push(' (bytes = source allocated by the render loop; deliv% = 0 means the hard ceiling dropped the section)'); out.push(' (* = clipped: some source in this file was elided, windowed, or its section dropped)'); diff --git a/src/mcp/explore-session-state.ts b/src/mcp/explore-session-state.ts index 7bcdf5a..26439c0 100644 --- a/src/mcp/explore-session-state.ts +++ b/src/mcp/explore-session-state.ts @@ -70,6 +70,13 @@ export interface ExploreFileEmission { ranges: ExploreLineRange[]; /** Source chars emitted for this file (excludes headers / fences). */ bytes: number; + /** + * Identity of the bytes those ranges were sliced from (CG-18). Cross-call + * dedup withholds a span only when the file still hashes to this, so an edit + * between two calls re-serves instead of pointing at source the agent holds a + * now-wrong copy of. Absent = unprovable, which dedup treats as "re-serve". + */ + fingerprint?: string; /** Set when ranges were dropped to stay under the per-file bound. */ rangesTruncated?: boolean; } @@ -309,6 +316,7 @@ export class ExploreSessionState { .map((f) => { const { ranges, truncated } = coalesceRanges(f.ranges ?? []); const out: ExploreFileEmission = { path: f.path, ranges, bytes: Math.max(0, f.bytes || 0) }; + if (typeof f.fingerprint === 'string' && f.fingerprint) out.fingerprint = f.fingerprint; if (truncated) out.rangesTruncated = true; return out; }); diff --git a/src/mcp/server-instructions.ts b/src/mcp/server-instructions.ts index bf3839a..7c6c6ce 100644 --- a/src/mcp/server-instructions.ts +++ b/src/mcp/server-instructions.ts @@ -62,6 +62,8 @@ calls; a grep/read exploration is dozens. - **After editing, check the staleness banner.** When a tool response starts with "⚠️ Some files referenced below were edited since the last index sync…", the listed files are pending re-index — Read those specific files for accurate content. Every file NOT in that banner is fresh, so still trust codegraph. A different, rarer banner — "⚠️ CodeGraph auto-sync is DISABLED…" — means live watching stopped entirely (the whole index is frozen, not just a few files); until it's resolved, Read files directly to confirm anything that may have changed. - **A file flagged "⚠ changed on disk after the last index sync" drifted from its index** (most common on projects queried via \`projectPath\`, which have no live watcher). Codegraph never serves a possibly-mis-sliced body from such a file — it either shows the file's full CURRENT source (trust it as a Read) or omits the source with this flag. When the source was omitted, Read that specific file; line numbers referencing it elsewhere in the response may be shifted until that project's next sync. All unflagged files remain trustworthy. +- **"Already sent earlier in this conversation" is a pointer, not a gap.** When a file's section carries that line instead of (or above) its source, an earlier \`codegraph_explore\` in THIS conversation already returned those exact lines and the file has not changed since — so the copy already in your context is current and exact. Scroll back to it; don't re-fetch it and don't Read the file. The bytes it freed went into source you have not seen yet, elsewhere in the same response. + ## Limitations - If a tool reports a project isn't indexed (no \`.codegraph/\`), stop calling codegraph tools for that project for the rest of the session and use your built-in tools there instead. Indexing is the user's decision — mention they can run \`codegraph init\` if it comes up, but don't run it yourself. diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 1493516..4a6e1a5 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -52,6 +52,16 @@ import { type ExploreFileEmission, type ExploreLineRange, } from './explore-session-state'; +import { + EXPLORE_DEDUP, + dedupeRange, + exploreDedupEnabled, + fileFingerprint, + formatBackReference, + mergeRanges, + servedRangesForFile, + symbolsInSpans, +} from './explore-dedup'; /** * An expected, recoverable "codegraph can't serve this" condition — most @@ -3120,26 +3130,50 @@ export class ToolHandler { // byte-identical. It only OBSERVES: it must never feed back into rendering. const diag = ExploreDiagnostics.start(query, projectRoot, budget, maxFiles, indexedFileCount); - // What this session has already been served for THIS project (CG-17). - // Read-only at this stage — it is reported in the diagnostic and nothing - // else, so the response is unchanged. Cross-call dedup (CG-18) and budget - // decay (CG-19) are the consumers this exists for. + // What this session has already been served for THIS project (CG-17), and + // whether this call may act on it (CG-18). Dedup is off on the session's + // first call by construction — there is nothing to point back AT — and off + // entirely under `CODEGRAPH_EXPLORE_DEDUP=0`. const priorCalls = viewForProject(readExploreSessionView(args), projectRoot); diag?.noteSession(priorCalls); + const dedupEnabled = exploreDedupEnabled() && (priorCalls?.calls.length ?? 0) > 0; + + // Cross-call dedup accounting (CG-18). `newSourceChars` is the load-bearing + // one: a response whose source is ENTIRELY back-references is the shape that + // reads as a failure, so the loop keeps the top suppressed file's real + // section in hand and restores it if nothing new made it in — see + // `suppressedFallback` below. + let newSourceChars = 0; + const backReferencedFiles: string[] = []; // What this call ends up emitting, per file — the record handed back to the // session state on the main thread. Filled by every render path below, then // filtered to the files that SURVIVE the final hard-ceiling cut, so the // record is what the agent actually received rather than what the loop // hoped to send. - const emittedByFile = new Map(); - const noteEmitted = (fp: string, ranges: ExploreLineRange[], bytes: number): void => { + // + // Back-referenced spans are recorded too, with zero bytes (CG-18): the + // record means "source the agent HOLDS for this file", not "bytes this call + // spent". Re-recording them refreshes them inside the retained-call window, + // so a file pointed at across many calls doesn't age out of the history and + // get re-served for no reason. + const emittedByFile = new Map< + string, + { ranges: ExploreLineRange[]; bytes: number; fingerprint?: string } + >(); + const noteEmitted = ( + fp: string, + ranges: ExploreLineRange[], + bytes: number, + fingerprint?: string, + ): void => { const existing = emittedByFile.get(fp); if (existing) { existing.ranges.push(...ranges); existing.bytes += bytes; + if (fingerprint) existing.fingerprint = fingerprint; } else { - emittedByFile.set(fp, { ranges: [...ranges], bytes }); + emittedByFile.set(fp, { ranges: [...ranges], bytes, fingerprint }); } }; @@ -3976,6 +4010,28 @@ export class ToolHandler { // instead of a different symbol's code under the requested name. const staleRendered: string[] = []; const staleOmitted: string[] = []; + // Anti-abandonment hold-back (CG-18). The first file dedup suppressed + // ENTIRELY, kept with its real section so it can be put back if the loop + // ends with no new source anywhere. A response made only of pointers is the + // shape that reads as "codegraph has nothing" — and one such response early + // in a session is enough to make an agent stop calling the tool at all — so + // the highest-ranked suppressed file is restored rather than risk it. It + // costs a re-serve of one file, on the one call shape where dedup would + // otherwise have saved everything: the safe direction. + type SuppressedFallback = { + filePath: string; + /** Index in `lines` where this file's pointer block starts. */ + at: number; + /** How many `lines` entries the pointer block occupies. */ + replacing: number; + /** The full, undeduped section to splice back in. */ + section: string[]; + sourceChars: number; + overhead: number; + ranges: ExploreLineRange[]; + fingerprint: string; + }; + let suppressedFallback: SuppressedFallback | null = null; // Reservation carry-forward (CG-21). A reservation is a promise the render // loop has to KEEP, not a cap it may quietly under-use: a file that cannot // spend what it was given — thin matched-symbol set, unreadable, drifted off @@ -4053,6 +4109,135 @@ export class ToolHandler { const fileLines = fileContent.split('\n'); const lang = group.nodes[0]?.language || ''; + const withLineNumbers = exploreLineNumbersEnabled(); + // Language-neutral separator between two non-contiguous slices of one file + // (no `//` — not a comment in Python, Ruby, etc.). With line numbers on, + // the line-number jump also signals the gap. + const GAP_MARKER = '\n\n... (gap) ...\n\n'; + + // Cross-call dedup (CG-18). `served` is what THIS session already sent the + // agent for THIS file, and it is empty unless the file still hashes to the + // bytes those spans were sliced from — an edit between calls means the + // agent's copy is wrong, so nothing is withheld. Every render path below + // routes its spans through `dedupeSpans`, which is the only place a span + // is ever dropped. + const fingerprint = fileFingerprint(fileContent); + const served = dedupEnabled ? servedRangesForFile(priorCalls, filePath, fingerprint) : []; + /** Render one line span exactly as the render paths do. */ + const renderSpan = (r: ExploreLineRange): string => { + const slice = fileLines.slice(r.start - 1, r.end).join('\n'); + return withLineNumbers ? numberSourceLines(slice, r.start) : slice; + }; + /** + * Apply the session history to a set of spans-with-text. A span the agent + * already holds is dropped and reported in `covered`; a partially-held one + * is re-rendered down to its new lines. Text is rebuilt from the surviving + * spans rather than sliced out of the original string — the spans ARE the + * contract with the session record, so rebuilding from them is what keeps + * what we claim to have sent and what we sent the same thing. + */ + const dedupeSpans = ( + parts: ReadonlyArray<{ range: ExploreLineRange; text: string }>, + ): { parts: Array<{ range: ExploreLineRange; text: string }>; covered: ExploreLineRange[] } => { + if (served.length === 0) return { parts: [...parts], covered: [] }; + const kept: Array<{ range: ExploreLineRange; text: string }> = []; + const covered: ExploreLineRange[] = []; + for (const part of parts) { + const split = dedupeRange(part.range, served); + if (split.covered.length === 0) { + kept.push(part); + continue; + } + covered.push(...split.covered); + for (const r of split.emit) kept.push({ range: r, text: renderSpan(r) }); + } + return { parts: kept, covered: mergeRanges(covered) }; + }; + const coveredChars = (spans: ReadonlyArray): number => + spans.reduce((sum, r) => sum + fileLines.slice(r.start - 1, r.end).join('\n').length, 0); + /** + * Emit one file's section — header, the back-reference for whatever the + * agent already holds, and the fence for what is new. Every render path + * ends here so that the dedup bookkeeping (freed bytes, freed `maxFiles` + * slot, the session record, the diagnostic) is written in exactly one + * place and no path can forget a piece of it. + * + * A fully-held file emits its header and pointer and NO fence, and + * deliberately does not consume a `maxFiles` slot: that is half of where + * the reclaimed budget goes (the other half is `sourceSpent`, which the + * carry-forward pool hands down the rank order). Both send bytes to files + * the agent has NOT seen, which is the whole point. + */ + const emitFileSection = (opts: { + header: string; + /** Deduped source. Empty ⇒ the agent already holds all of it. */ + body: string; + /** Spans `body` covers. */ + ranges: ExploreLineRange[]; + /** Spans replaced by the back-reference. */ + covered: ExploreLineRange[]; + /** Chars charged to `totalChars` on top of the body (fences, header). */ + overhead: number; + mode: 'whole' | 'clusters' | 'focused' | 'skeleton'; + clipped: boolean; + /** The undeduped render, kept for the no-new-source fallback. */ + fullBody: string; + fullRanges: ExploreLineRange[]; + }): void => { + // A remainder too small to be worth a fence is folded into the pointer + // (see MIN_DELTA_CHARS). Its ranges are then NOT recorded — the record + // must only ever claim source that was actually sent. + const folded = opts.covered.length > 0 && opts.body.length < EXPLORE_DEDUP.MIN_DELTA_CHARS; + const body = folded ? '' : opts.body; + const ranges = folded ? [] : opts.ranges; + const at = lines.length; + lines.push(opts.header, ''); + if (opts.covered.length > 0) { + const pointer = formatBackReference( + filePath, + opts.covered, + symbolsInSpans(group.nodes, opts.covered), + { partial: body.length > 0 }, + ); + lines.push(pointer, ''); + totalChars += pointer.length + 2; + backReferencedFiles.push(filePath); + } + if (body.length > 0) { + lines.push('```' + lang, body, '```', ''); + totalChars += body.length + opts.overhead; + sourceSpent += body.length; + newSourceChars += body.length; + diag?.recordRender(filePath, opts.mode, body.length, opts.clipped || opts.covered.length > 0); + if (opts.covered.length > 0) diag?.recordDedup(filePath, coveredChars(opts.covered), opts.covered); + noteEmitted(filePath, [...ranges, ...opts.covered], body.length, fingerprint); + renderedFilePaths.push(filePath); + filesIncluded++; + return; + } + // Fully held. The section is the pointer; the slot and the bytes go to a + // file the agent has not seen. The spans are still recorded (at zero + // bytes) because the record means "source the agent HAS", not "bytes + // this call spent" — refreshing them keeps a long session from ageing + // them out of the retained window and re-serving them for nothing. + totalChars += opts.overhead; + diag?.recordRender(filePath, 'backref', 0, false); + diag?.recordDedup(filePath, coveredChars(opts.covered), opts.covered); + noteEmitted(filePath, opts.covered, 0, fingerprint); + renderedFilePaths.push(filePath); + if (!suppressedFallback && opts.fullBody.length > 0) { + suppressedFallback = { + filePath, + at, + replacing: lines.length - at, + section: [opts.header, '', '```' + lang, opts.fullBody, '```', ''], + sourceChars: opts.fullBody.length, + overhead: opts.overhead, + ranges: opts.fullRanges, + fingerprint, + }; + } + }; // Disk-drift gate (#1474): every render branch below except whole-file // slices fileContent (CURRENT bytes) at INDEXED line ranges. Content is @@ -4132,8 +4317,7 @@ export class ToolHandler { // Pass 2: render in line order — full body for chosen symbols, else the // signature line (capped, with a "+N more" tail so the structure map of a // god-file doesn't itself bloat the budget). - const skel: string[] = []; - const skelRanges: ExploreLineRange[] = []; + const skel: Array<{ range: ExploreLineRange; text: string }> = []; let coveredUntil = 0; // skip symbols already inside an emitted body let sigCount = 0, sigDropped = 0; const SIG_MAX = Math.max(12, budget.maxSymbolsInFileHeader * 2); @@ -4142,8 +4326,10 @@ export class ToolHandler { if (bodyIds.has(n.id)) { const end = n.endLine; const body = fileLines.slice(n.startLine - 1, end).join('\n'); - skel.push(exploreLineNumbersEnabled() ? numberSourceLines(body, n.startLine) : body); - skelRanges.push({ start: n.startLine, end }); + skel.push({ + range: { start: n.startLine, end }, + text: withLineNumbers ? numberSourceLines(body, n.startLine) : body, + }); coveredUntil = end; } else { // Elide the body, emit the signature. node.startLine can point at a @@ -4156,13 +4342,15 @@ export class ToolHandler { if (sigCount >= SIG_MAX) { sigDropped++; continue; } const sig = (fileLines[lineNo - 1] || '').trim(); if (sig) { - skel.push(exploreLineNumbersEnabled() ? `${lineNo}\t${sig}` : sig); - skelRanges.push({ start: lineNo, end: lineNo }); + skel.push({ + range: { start: lineNo, end: lineNo }, + text: withLineNumbers ? `${lineNo}\t${sig}` : sig, + }); sigCount++; } } } - if (sigDropped > 0) skel.push(`… +${sigDropped} more (signatures elided)`); + const sigTail = sigDropped > 0 ? `… +${sigDropped} more (signatures elided)` : ''; if (skel.length > 0) { const names = [...new Set(group.nodes.filter(n => n.kind !== 'import' && n.kind !== 'export').map(n => n.name))] .slice(0, budget.maxSymbolsInFileHeader).join(', '); @@ -4175,14 +4363,25 @@ export class ToolHandler { const tag = bodyIds.size > 0 ? 'focused (the methods you named in full, the rest as signatures — codegraph_explore a signature by name for its body; do NOT Read)' : 'skeleton (signatures only — codegraph_explore a name for its full body; do NOT Read)'; - lines.push(fileSectionHeader(filePath, `${names} · ${tag}`), '', '```' + lang, skel.join('\n'), '```', ''); - totalChars += skel.join('\n').length + 120; - sourceSpent += skel.join('\n').length; - // Always "clipped": the per-symbol view elides bodies by construction. - diag?.recordRender(filePath, bodyIds.size > 0 ? 'focused' : 'skeleton', skel.join('\n').length, true); - noteEmitted(filePath, skelRanges, skel.join('\n').length); - renderedFilePaths.push(filePath); - filesIncluded++; + // Dedup runs on the per-symbol parts, so a body the agent already has + // becomes a pointer while the signature map around it survives intact + // (a one-line signature is far under MIN_COVERED_LINES and is never + // withheld — the structure map is what makes this render legible). + const dd = dedupeSpans(skel); + const withTail = (parts: ReadonlyArray<{ text: string }>) => + [...parts.map((p) => p.text), ...(sigTail ? [sigTail] : [])].join('\n'); + emitFileSection({ + header: fileSectionHeader(filePath, `${names} · ${tag}`), + body: dd.parts.length > 0 ? withTail(dd.parts) : '', + ranges: dd.parts.map((p) => p.range), + covered: dd.covered, + overhead: 120, + mode: bodyIds.size > 0 ? 'focused' : 'skeleton', + // Always "clipped": the per-symbol view elides bodies by construction. + clipped: true, + fullBody: withTail(skel), + fullRanges: skel.map((p) => p.range), + }); continue; } } @@ -4260,7 +4459,14 @@ export class ToolHandler { && totalChars + fileContent.length + EXPLORE_ALLOCATION.FILE_OVERHEAD <= renderCeiling); if (fileLines.length <= WHOLE_FILE_MAX_LINES && buysWhole) { const body = fileContent.replace(/\n+$/, ''); - let wholeSection = exploreLineNumbersEnabled() ? numberSourceLines(body, 1) : body; + const wholeRange: ExploreLineRange = { start: 1, end: body.split('\n').length }; + const fullSection = withLineNumbers ? numberSourceLines(body, 1) : body; + // The buy decision above was made on the file's FULL size on purpose: it + // asks "did this file's relevance earn all of itself", which dedup does + // not change. Dedup then only ever makes the render smaller, so a buy + // that was funded stays funded. + const ddWhole = dedupeSpans([{ range: wholeRange, text: fullSection }]); + const wholeSection = ddWhole.parts.map((p) => p.text).join(GAP_MARKER); const uniqSymbols = [...new Set( group.nodes .filter(n => n.kind !== 'import' && n.kind !== 'export') @@ -4281,14 +4487,18 @@ export class ToolHandler { diag?.recordSkip(filePath, 'budget-whole-file'); continue; } - lines.push(wholeHeader, '', '```' + lang, wholeSection, '```', ''); - totalChars += wholeSection.length + 200; - sourceSpent += wholeSection.length; - diag?.recordRender(filePath, 'whole', wholeSection.length, false); - // The whole file, minus any trailing blank lines the render trimmed. - noteEmitted(filePath, [{ start: 1, end: body.split('\n').length }], wholeSection.length); - renderedFilePaths.push(filePath); - filesIncluded++; + emitFileSection({ + header: wholeHeader, + body: wholeSection, + // The whole file, minus any trailing blank lines the render trimmed. + ranges: ddWhole.parts.map((p) => p.range), + covered: ddWhole.covered, + overhead: 200, + mode: 'whole', + clipped: false, + fullBody: fullSection, + fullRanges: [wholeRange], + }); if (fileStale) staleRendered.push(filePath); continue; } @@ -4441,10 +4651,6 @@ export class ToolHandler { // until the per-file char cap is hit. Truly enormous single clusters // get tail-trimmed with a marker. const contextPadding = 3; - const withLineNumbers = exploreLineNumbersEnabled(); - // Language-neutral separator (no `//` — not a comment in Python, Ruby, - // etc.). With line numbers on, the line-number jump also signals the gap. - const GAP_MARKER = '\n\n... (gap) ...\n\n'; // An oversize spine method (the call path runs THROUGH a god-method — n8n's // processRunExecutionData is 962 lines) is windowed to its next-hop CALL site // plus the signature head, NOT dumped whole. Without this the cluster is too big @@ -4453,42 +4659,50 @@ export class ToolHandler { // the spine's call still appears in context. const OVERSIZE_SPINE_LINES = 200; const SPINE_WINDOW = 28; // lines each side of the next-hop call site - // Returns the rendered text AND the line spans it covers. The spans are - // what the session record is built from (CG-17): reporting them from the - // same function that slices the source is what keeps the record honest — - // a second function mirroring these window/padding rules would drift, and - // a record that claims lines it never sent withholds them from a later - // call, which costs a Read. + // Returns the rendered text as SPAN-KEYED PARTS. Every part carries the + // exact line range its text was sliced from, which two things depend on: + // the session record (CG-17) — a record claiming lines it never sent would + // withhold them from a later call, costing a Read — and cross-call dedup + // (CG-18), which rebuilds a part's text from a narrower span when the + // agent already holds the rest. Both read the spans from the function that + // does the slicing; a second function mirroring these window/padding rules + // would drift. + type SectionPart = { range: ExploreLineRange; text: string }; + const sectionText = (parts: ReadonlyArray): string => + parts.map((p) => p.text).join(GAP_MARKER); const buildSection = ( c: { start: number; end: number; hasSpine?: boolean; spineCallLine?: number }, - ): { text: string; ranges: ExploreLineRange[] } => { + ): SectionPart[] => { if (c.hasSpine && c.spineCallLine && (c.end - c.start + 1) > OVERSIZE_SPINE_LINES) { const call = c.spineCallLine; const winStart = Math.max(c.start, call - SPINE_WINDOW); const winEnd = Math.min(c.end, call + SPINE_WINDOW); - const parts: string[] = []; - const spans: ExploreLineRange[] = []; + const parts: SectionPart[] = []; // Signature head, only when it sits clearly above the window (else the // window already covers the method opening). const headEnd = Math.min(c.start + 4, winStart - 2); if (headEnd >= c.start) { const head = fileLines.slice(c.start - 1, headEnd).join('\n'); - parts.push(withLineNumbers ? numberSourceLines(head, c.start) : head); - spans.push({ start: c.start, end: headEnd }); + parts.push({ + range: { start: c.start, end: headEnd }, + text: withLineNumbers ? numberSourceLines(head, c.start) : head, + }); } const win = fileLines.slice(winStart - 1, winEnd).join('\n'); - parts.push(withLineNumbers ? numberSourceLines(win, winStart) : win); - spans.push({ start: winStart, end: winEnd }); - return { text: parts.join(GAP_MARKER), ranges: spans }; + parts.push({ + range: { start: winStart, end: winEnd }, + text: withLineNumbers ? numberSourceLines(win, winStart) : win, + }); + return parts; } const startIdx = Math.max(0, c.start - 1 - contextPadding); const endIdx = Math.min(fileLines.length, c.end + contextPadding); const slice = fileLines.slice(startIdx, endIdx).join('\n'); // startIdx is 0-based, so the slice's first line is line startIdx + 1. - return { + return [{ + range: { start: startIdx + 1, end: endIdx }, text: withLineNumbers ? numberSourceLines(slice, startIdx + 1) : slice, - ranges: [{ start: startIdx + 1, end: endIdx }], - }; + }]; }; /** @@ -4507,10 +4721,7 @@ export class ToolHandler { * body is never cut, and the members are chosen by the same importance the * cluster ranking uses. Returns null when nothing needed shrinking. */ - const shrinkCluster = ( - c: ExploreCluster, - cap: number, - ): { text: string; ranges: ExploreLineRange[] } | null => { + const shrinkCluster = (c: ExploreCluster, cap: number): SectionPart[] | null => { if (c.members.length < 2) return null; const byImportance = [...c.members].sort((a, b) => b.importance - a.importance || (a.end - a.start) - (b.end - b.start) || a.start - b.start); @@ -4535,11 +4746,30 @@ export class ToolHandler { if (last && r.start <= last.end + gapThreshold) last.end = Math.max(last.end, r.end); else merged.push({ start: r.start, end: r.end }); } - const sections = merged.map((m) => buildSection(m)); - return { - text: sections.map((s) => s.text).join(GAP_MARKER), - ranges: sections.flatMap((s) => s.ranges), - }; + return merged.flatMap((m) => buildSection(m)); + }; + + /** + * One cluster's final parts: built, shrunk if it overruns `cap`, then + * passed through the session history (CG-18). + * + * The shrink decision reads the DEDUPED length on purpose. A cluster whose + * bytes the agent already holds costs this response nothing, so shrinking + * it on its raw size would drop new symbols to make room for source that + * is not being sent — spending the file's budget on nothing. + */ + const renderCluster = ( + c: ExploreCluster, + cap: number, + ): { parts: SectionPart[]; covered: ExploreLineRange[]; shrunk: boolean } => { + const base = dedupeSpans(buildSection(c)); + if (sectionText(base.parts).length <= cap) { + return { parts: base.parts, covered: base.covered, shrunk: false }; + } + const shrunk = shrinkCluster(c, cap); + if (shrunk === null) return { parts: base.parts, covered: base.covered, shrunk: false }; + const dd = dedupeSpans(shrunk); + return { parts: dd.parts, covered: dd.covered, shrunk: true }; }; // Rank clusters for inclusion under the per-file cap. Entry-point @@ -4586,23 +4816,28 @@ export class ToolHandler { // spine can't run away or starve co-flow files entirely. const SPINE_CEILING = Math.min(Math.round(allowance * 1.5), headroom); const chosenIndices = new Set(); - // Shrunk renders for oversize clusters, by cluster index (CG-12). Computed + // Final renders (deduped, shrunk where oversize) by cluster index. Computed // during selection and reused at emission so the two never disagree. - const shrunkSections = new Map(); + const renderedClusters = new Map>(); + let anyClusterShrunk = false; let projectedChars = 0; for (const rc of rankedClusters) { - const sectionLen = buildSection(rc.c).text.length + (chosenIndices.size > 0 ? GAP_MARKER.length : 0); // The top-ranked cluster is always taken — an empty file section sends the // agent to Read, negating the savings. But "always taken" is not "taken at // any size": when it overruns the reservation it is SHRUNK to the // highest-importance whole symbol ranges inside it, so a single-cluster - // god-file spends its allotment instead of the whole response's. - if (chosenIndices.size === 0) { - const cap = rc.c.hasSpine ? SPINE_CEILING : fileBudget; - const shrunk = sectionLen > cap ? shrinkCluster(rc.c, cap) : null; - if (shrunk !== null) shrunkSections.set(rc.idx, shrunk); + // god-file spends its allotment instead of the whole response's. Later + // clusters are never shrunk — they either fit or wait for another call. + const first = chosenIndices.size === 0; + const cap = rc.c.hasSpine ? SPINE_CEILING : fileBudget; + const section = renderCluster(rc.c, first ? cap : Infinity); + const text = sectionText(section.parts); + const sectionLen = text.length + (!first && text.length > 0 ? GAP_MARKER.length : 0); + if (first) { + renderedClusters.set(rc.idx, section); + anyClusterShrunk = anyClusterShrunk || section.shrunk; chosenIndices.add(rc.idx); - projectedChars += shrunk !== null ? shrunk.text.length : sectionLen; + projectedChars += sectionLen; continue; } // A spine cluster (the rendered call path) is the flow answer — include it @@ -4611,6 +4846,7 @@ export class ToolHandler { const fits = projectedChars + sectionLen <= fileBudget; const spineFits = rc.c.hasSpine && projectedChars + sectionLen <= SPINE_CEILING; if (!fits && !spineFits) continue; + renderedClusters.set(rc.idx, section); chosenIndices.add(rc.idx); projectedChars += sectionLen; } @@ -4619,13 +4855,18 @@ export class ToolHandler { let fileSection = ''; const allSymbols: string[] = []; const sectionRanges: ExploreLineRange[] = []; + const coveredRanges: ExploreLineRange[] = []; for (let i = 0; i < clusters.length; i++) { if (!chosenIndices.has(i)) continue; const cluster = clusters[i]!; - const section = shrunkSections.get(i) ?? buildSection(cluster); - if (fileSection.length > 0) fileSection += GAP_MARKER; - fileSection += section.text; - sectionRanges.push(...section.ranges); + const section = renderedClusters.get(i)!; + const text = sectionText(section.parts); + if (text.length > 0) { + if (fileSection.length > 0) fileSection += GAP_MARKER; + fileSection += text; + } + sectionRanges.push(...section.parts.map((p) => p.range)); + coveredRanges.push(...section.covered); allSymbols.push(...cluster.symbols); } @@ -4635,7 +4876,7 @@ export class ToolHandler { // method is useless (the agent just Reads the rest for the other half), which // is the very fallback explore exists to prevent. A pathological file is // bounded by the cluster SELECTION above + the total hard ceiling. - if (chosenIndices.size < clusters.length || shrunkSections.size > 0) { + if (chosenIndices.size < clusters.length || anyClusterShrunk) { anyFileTrimmed = true; } @@ -4673,19 +4914,62 @@ export class ToolHandler { continue; } - lines.push(fileHeader); - lines.push(''); - lines.push('```' + lang); - lines.push(fileSection); - lines.push('```'); - lines.push(''); + // The undeduped render of the same clusters, needed only if this file ends + // up fully back-referenced AND the whole call finds nothing new to say — + // see `suppressedFallback`. Built lazily: on every other call it is dead + // weight. + const fullClusterParts = fileSection.length === 0 + ? clusters.flatMap((c, i) => (chosenIndices.has(i) ? buildSection(c) : [])) + : []; + emitFileSection({ + header: fileHeader, + body: fileSection, + ranges: sectionRanges, + covered: mergeRanges(coveredRanges), + overhead: 200, + mode: 'clusters', + clipped: chosenIndices.size < clusters.length, + fullBody: sectionText(fullClusterParts), + fullRanges: fullClusterParts.map((p) => p.range), + }); + } - totalChars += fileSection.length + 200; - sourceSpent += fileSection.length; - diag?.recordRender(filePath, 'clusters', fileSection.length, chosenIndices.size < clusters.length); - noteEmitted(filePath, sectionRanges, fileSection.length); - renderedFilePaths.push(filePath); - filesIncluded++; + // Anti-abandonment restore (CG-18). Dedup withheld everything and nothing new + // took its place — the response would be pointers only, which is the shape + // that reads as "codegraph found nothing" and sends the agent to Read for + // good. Put the top suppressed file back, in full, and keep its pointer off. + // Deliberately checked against `newSourceChars` (source THIS call emitted) + // rather than the response length: the flow and blast-radius sections are + // always there, and they are not what makes a response feel sufficient. + // Cast, not annotation: the only writer is the render loop's `emitFileSection` + // closure, which TypeScript's flow analysis cannot see, so it narrows the + // variable to `null` here and the truthiness check below would be `never`. + const restore = suppressedFallback as SuppressedFallback | null; + if (newSourceChars === 0 && restore) { + if (totalChars + restore.sourceChars + restore.overhead <= renderCeiling) { + lines.splice(restore.at, restore.replacing, ...restore.section); + totalChars += restore.sourceChars + restore.overhead; + sourceSpent += restore.sourceChars; + newSourceChars += restore.sourceChars; + filesIncluded++; + const idx = backReferencedFiles.indexOf(restore.filePath); + if (idx >= 0) backReferencedFiles.splice(idx, 1); + emittedByFile.set(restore.filePath, { + ranges: [...restore.ranges], + bytes: restore.sourceChars, + fingerprint: restore.fingerprint, + }); + diag?.recordRender(restore.filePath, 'clusters', restore.sourceChars, false); + diag?.recordDedup(restore.filePath, 0, []); + } + } + + // The back-reference convention, stated once where the verbatim guarantee is + // (#1474 does the same for drift). Without it a pointer reads as an + // apology for missing source rather than as an index into source the agent + // already has. + if (backReferencedFiles.length > 0) { + lines[verbatimHeaderIdx] += ` (Files marked **"Already sent earlier in this conversation"** are not repeated: their source came back on an earlier codegraph_explore call in THIS conversation and the file has not changed since, so that copy is exact and current — scroll back for it rather than re-fetching or Reading.)`; } // Drift epilogue (#1474). The "verbatim / do not Read" guarantee above @@ -4833,12 +5117,22 @@ export class ToolHandler { // Session record (CG-17): only the files that SURVIVED the hard ceiling — // a section the truncation dropped was never delivered, and recording it // would let a later call withhold source the agent has never seen. + // + // A back-referenced file records its spans at ZERO bytes (CG-18) — it is + // still source the agent holds for this file, which is what the record + // means; dropping it would let the span age out of the retained window and + // be re-served for nothing. const emittedFiles: ExploreFileEmission[] = []; let sourceBytes = 0; for (const fp of survivors) { const emitted = emittedByFile.get(fp); - if (!emitted || emitted.bytes <= 0) continue; - emittedFiles.push({ path: fp, ranges: emitted.ranges, bytes: emitted.bytes }); + if (!emitted || emitted.ranges.length === 0) continue; + emittedFiles.push({ + path: fp, + ranges: emitted.ranges, + bytes: emitted.bytes, + fingerprint: emitted.fingerprint, + }); sourceBytes += emitted.bytes; } return this.exploreResult(finalText, {