feat(explore): point at source this session already sent, don't send it twice (CG-18)
A later explore call re-served whatever it re-ranked, so on the #1500 report the 4th call spent its envelope on the spine the 1st call had already delivered. CG-17 recorded what was served; this acts on it. What a withheld span becomes is the whole design: a POINTER, never a silence. An insufficient-feeling response is what sends an agent to Read, and one or two of those early in a session teach it to abandon codegraph — so the replacement names the file, the symbols and the line spans, and says both that the source came from THIS conversation and that the file has not changed since. - Content fingerprint, not the drift flag, gates it. They answer different questions: two calls inside one drift window served the same current bytes, while a file edited AND re-synced between calls is never "stale" and yet the agent's copy is now wrong. An edited file re-emits in full. - Only a covered run of >= 8 lines is replaced, and a remainder under 160 chars folds into the pointer. Below those the pointer costs more than the source and the block reads as shredded — a fence holding `228\t` is a broken-looking response, which is the expensive failure. - The reclaimed bytes go to files the agent has NOT seen, two ways: a smaller `sourceSpent` hands slack down CG-21's carry-forward pool, and a fully back-referenced file gives up its maxFiles slot the way a cliffed one does. Within a file, the cluster shrink now reads the DEDUPED length, so it never drops new symbols to make room for source it isn't sending. - If dedup suppresses everything and nothing new takes its place, the top suppressed file is spliced back in whole. An all-pointer response is the shape that reads as "codegraph found nothing"; one re-served file is the cheaper mistake. Kill switch: CODEGRAPH_EXPLORE_DEDUP=0.
This commit is contained in:
@@ -0,0 +1,360 @@
|
|||||||
|
/**
|
||||||
|
* Cross-call source dedup (CG-18).
|
||||||
|
*
|
||||||
|
* A later `codegraph_explore` call in a session must not re-send source an
|
||||||
|
* earlier call already delivered — but every byte it withholds has to be
|
||||||
|
* replaced by a POINTER, never a silence. That asymmetry is what this suite
|
||||||
|
* guards, because the two failure directions cost wildly different amounts: a
|
||||||
|
* duplicate range wastes a few thousand chars, while a response that reads as
|
||||||
|
* "codegraph doesn't have it" costs a Read — and one or two of those early in a
|
||||||
|
* session teach an agent to stop calling the tool at all.
|
||||||
|
*
|
||||||
|
* Three layers:
|
||||||
|
* 1. the range algebra — what is withheld, and the thresholds that stop it
|
||||||
|
* from shredding a block into slivers;
|
||||||
|
* 2. the fingerprint gate — an edit between two calls must re-emit, since a
|
||||||
|
* pointer to pre-edit source is worse than no dedup at all;
|
||||||
|
* 3. the handler seam — a real second call against a real index: no duplicate
|
||||||
|
* ranges, a pointer for everything withheld, the reclaimed budget spent on
|
||||||
|
* source the agent has NOT seen, and never an all-pointer response.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||||
|
import * as fs from 'fs';
|
||||||
|
import * as path from 'path';
|
||||||
|
import * as os from 'os';
|
||||||
|
import CodeGraph from '../src/index';
|
||||||
|
import { ToolHandler } from '../src/mcp/tools';
|
||||||
|
import { ExploreSessionState, type ExploreProjectState } from '../src/mcp/explore-session-state';
|
||||||
|
import {
|
||||||
|
EXPLORE_DEDUP,
|
||||||
|
dedupeRange,
|
||||||
|
fileFingerprint,
|
||||||
|
formatBackReference,
|
||||||
|
intersectRange,
|
||||||
|
mergeRanges,
|
||||||
|
servedRangesForFile,
|
||||||
|
subtractRange,
|
||||||
|
symbolsInSpans,
|
||||||
|
} from '../src/mcp/explore-dedup';
|
||||||
|
|
||||||
|
const FIXTURE_SRC = path.join(__dirname, 'fixtures', 'payroll-go');
|
||||||
|
const QUERY = 'how does payroll cycle create and calculate payslips?';
|
||||||
|
const POINTER = 'Already sent earlier in this conversation';
|
||||||
|
|
||||||
|
/** A prior-state shaped like the session tracker's, for the algebra tests. */
|
||||||
|
function prior(files: Array<{ path: string; ranges: Array<[number, number]>; fingerprint?: string }>): ExploreProjectState {
|
||||||
|
return {
|
||||||
|
projectRoot: '/repo',
|
||||||
|
callCount: 1,
|
||||||
|
responseBytes: 1000,
|
||||||
|
calls: [{
|
||||||
|
index: 1,
|
||||||
|
projectRoot: '/repo',
|
||||||
|
query: 'q',
|
||||||
|
sourceBytes: 500,
|
||||||
|
responseBytes: 1000,
|
||||||
|
files: files.map((f) => ({
|
||||||
|
path: f.path,
|
||||||
|
ranges: f.ranges.map(([start, end]) => ({ start, end })),
|
||||||
|
bytes: 500,
|
||||||
|
fingerprint: f.fingerprint,
|
||||||
|
})),
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('range algebra', () => {
|
||||||
|
it('subtracts a held span out of the middle of an intended one', () => {
|
||||||
|
expect(subtractRange({ start: 1, end: 100 }, [{ start: 20, end: 40 }]))
|
||||||
|
.toEqual([{ start: 1, end: 19 }, { start: 41, end: 100 }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('subtracts held spans at either edge, and a full cover to nothing', () => {
|
||||||
|
expect(subtractRange({ start: 10, end: 50 }, [{ start: 1, end: 20 }]))
|
||||||
|
.toEqual([{ start: 21, end: 50 }]);
|
||||||
|
expect(subtractRange({ start: 10, end: 50 }, [{ start: 30, end: 90 }]))
|
||||||
|
.toEqual([{ start: 10, end: 29 }]);
|
||||||
|
expect(subtractRange({ start: 10, end: 50 }, [{ start: 1, end: 90 }])).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('intersects to exactly what both sides hold', () => {
|
||||||
|
expect(intersectRange({ start: 10, end: 50 }, [{ start: 1, end: 20 }, { start: 45, end: 80 }]))
|
||||||
|
.toEqual([{ start: 10, end: 20 }, { start: 45, end: 50 }]);
|
||||||
|
expect(intersectRange({ start: 10, end: 50 }, [{ start: 60, end: 80 }])).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('merges touching spans — two adjacent blocks are one block of source', () => {
|
||||||
|
expect(mergeRanges([{ start: 5, end: 9 }, { start: 10, end: 12 }, { start: 40, end: 41 }]))
|
||||||
|
.toEqual([{ start: 5, end: 12 }, { start: 40, end: 41 }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('emits ONLY the delta when a later call wants a wider window', () => {
|
||||||
|
// Call 1 sent the method; call 2 wants the class around it.
|
||||||
|
const { emit, covered } = dedupeRange({ start: 80, end: 200 }, [{ start: 100, end: 140 }]);
|
||||||
|
expect(covered).toEqual([{ start: 100, end: 140 }]);
|
||||||
|
expect(emit).toEqual([{ start: 80, end: 99 }, { start: 141, end: 200 }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('withholds nothing when the overlap is smaller than a chunk worth pointing at', () => {
|
||||||
|
// Context padding and signature lines land here. Replacing them costs more
|
||||||
|
// in pointer text than the source is worth, and shreds the block.
|
||||||
|
const overlap = EXPLORE_DEDUP.MIN_COVERED_LINES - 1;
|
||||||
|
const { emit, covered } = dedupeRange({ start: 1, end: 100 }, [{ start: 10, end: 10 + overlap - 1 }]);
|
||||||
|
expect(covered).toEqual([]);
|
||||||
|
expect(emit).toEqual([{ start: 1, end: 100 }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('leaves an untouched span exactly as it was', () => {
|
||||||
|
expect(dedupeRange({ start: 1, end: 50 }, [{ start: 200, end: 400 }]))
|
||||||
|
.toEqual({ emit: [{ start: 1, end: 50 }], covered: [] });
|
||||||
|
expect(dedupeRange({ start: 1, end: 50 }, []))
|
||||||
|
.toEqual({ emit: [{ start: 1, end: 50 }], covered: [] });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('the fingerprint gate', () => {
|
||||||
|
const FP = fileFingerprint('package main\nfunc main() {}\n');
|
||||||
|
|
||||||
|
it('returns the spans a call served for a file whose bytes are unchanged', () => {
|
||||||
|
const state = prior([{ path: 'a.go', ranges: [[1, 40], [60, 80]], fingerprint: FP }]);
|
||||||
|
expect(servedRangesForFile(state, 'a.go', FP)).toEqual([{ start: 1, end: 40 }, { start: 60, end: 80 }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns NOTHING once the file has been edited — a pointer would be wrong', () => {
|
||||||
|
const state = prior([{ path: 'a.go', ranges: [[1, 40]], fingerprint: FP }]);
|
||||||
|
const edited = fileFingerprint('package main\nfunc main() { changed() }\n');
|
||||||
|
expect(servedRangesForFile(state, 'a.go', edited)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('ignores a record that cannot prove what it served', () => {
|
||||||
|
const state = prior([{ path: 'a.go', ranges: [[1, 40]] }]);
|
||||||
|
expect(servedRangesForFile(state, 'a.go', FP)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('never crosses files, and is empty for an untracked session', () => {
|
||||||
|
const state = prior([{ path: 'a.go', ranges: [[1, 40]], fingerprint: FP }]);
|
||||||
|
expect(servedRangesForFile(state, 'b.go', FP)).toEqual([]);
|
||||||
|
expect(servedRangesForFile(null, 'a.go', FP)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('distinguishes two files that hash the same prefix but differ in length', () => {
|
||||||
|
expect(fileFingerprint('abc')).not.toBe(fileFingerprint('abcd'));
|
||||||
|
expect(fileFingerprint('abc')).toBe(fileFingerprint('abc'));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('the back-reference itself', () => {
|
||||||
|
const covered = [{ start: 100, end: 240 }];
|
||||||
|
|
||||||
|
it('names the file, the span and the symbols, and says the copy is still good', () => {
|
||||||
|
const text = formatBackReference('internal/x.go', covered, ['RunCycle', 'BuildPayslip'], { partial: false });
|
||||||
|
expect(text).toContain('internal/x.go');
|
||||||
|
expect(text).toContain('L100-240');
|
||||||
|
expect(text).toContain('RunCycle, BuildPayslip');
|
||||||
|
expect(text).toContain(POINTER);
|
||||||
|
expect(text).toContain('unchanged on disk');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('never tells the agent to Read, in either shape', () => {
|
||||||
|
for (const partial of [true, false]) {
|
||||||
|
const text = formatBackReference('x.go', covered, ['A'], { partial });
|
||||||
|
expect(text).toMatch(/do NOT Read/i);
|
||||||
|
expect(text).not.toMatch(/\bRead (this|the) file (for|to)\b/i);
|
||||||
|
expect(text).not.toMatch(/omitted|unavailable|could not/i);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('says the block below is only the NEW lines when the call still sends some', () => {
|
||||||
|
expect(formatBackReference('x.go', covered, [], { partial: true })).toContain('NEW lines');
|
||||||
|
expect(formatBackReference('x.go', covered, [], { partial: false })).toContain('not repeated here');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('names only symbols that actually fall in the withheld spans', () => {
|
||||||
|
const nodes = [
|
||||||
|
{ name: 'InSpan', kind: 'function', startLine: 110, endLine: 130 },
|
||||||
|
{ name: 'Outside', kind: 'function', startLine: 300, endLine: 320 },
|
||||||
|
{ name: 'AnImport', kind: 'import', startLine: 105, endLine: 105 },
|
||||||
|
];
|
||||||
|
expect(symbolsInSpans(nodes, covered)).toEqual(['InSpan']);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('a second call against a real index', () => {
|
||||||
|
let testDir: string;
|
||||||
|
let cg: CodeGraph;
|
||||||
|
let handler: ToolHandler;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cg18-'));
|
||||||
|
fs.cpSync(FIXTURE_SRC, testDir, { recursive: true });
|
||||||
|
fs.rmSync(path.join(testDir, '.codegraph'), { recursive: true, force: true });
|
||||||
|
cg = CodeGraph.initSync(testDir);
|
||||||
|
await cg.indexAll();
|
||||||
|
handler = new ToolHandler(cg);
|
||||||
|
}, 120_000);
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
if (cg) cg.destroy();
|
||||||
|
if (testDir && fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
const explore = (query: string, session?: ExploreSessionState, args: Record<string, unknown> = {}) =>
|
||||||
|
handler.execute('codegraph_explore', { query, ...args }, session).then((r) => r.content[0]!.text);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The line numbers actually inside each file's fenced source. Read off the
|
||||||
|
* RESPONSE, not the bookkeeping — "no duplicate ranges" is a claim about what
|
||||||
|
* the agent received, and checking it against the record we also wrote would
|
||||||
|
* prove only that the two agree.
|
||||||
|
*/
|
||||||
|
function fencedLines(text: string): Map<string, Set<number>> {
|
||||||
|
const out = new Map<string, Set<number>>();
|
||||||
|
let current: string | null = null;
|
||||||
|
let inFence = false;
|
||||||
|
for (const line of text.split('\n')) {
|
||||||
|
const header = /^\*\*`([^`]+)`\*\*/.exec(line);
|
||||||
|
if (header && !inFence) { current = header[1]!; continue; }
|
||||||
|
if (!inFence && current && line.startsWith('```')) { inFence = true; continue; }
|
||||||
|
if (inFence && line === '```') { inFence = false; continue; }
|
||||||
|
if (!inFence || !current) continue;
|
||||||
|
const numbered = /^(\d+)\t/.exec(line);
|
||||||
|
if (!numbered) continue;
|
||||||
|
if (!out.has(current)) out.set(current, new Set());
|
||||||
|
out.get(current)!.add(Number(numbered[1]));
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
it('never re-sends a line it already sent, and points at every line it withholds', async () => {
|
||||||
|
const session = new ExploreSessionState();
|
||||||
|
const first = await explore(QUERY, session);
|
||||||
|
const second = await explore(QUERY, session);
|
||||||
|
|
||||||
|
const before = fencedLines(first);
|
||||||
|
const after = fencedLines(second);
|
||||||
|
expect(after.size).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
// Every file whose source the second call withheld carries a pointer, and
|
||||||
|
// the pointer names it.
|
||||||
|
expect(second).toContain(POINTER);
|
||||||
|
for (const [file, lines] of before) {
|
||||||
|
const repeated = [...(after.get(file) ?? [])].filter((n) => lines.has(n));
|
||||||
|
if (repeated.length === 0) continue;
|
||||||
|
// The only sanctioned repeat is the anti-abandonment restore, which fires
|
||||||
|
// ONLY when the call found nothing new to say — and this one did.
|
||||||
|
throw new Error(`call 2 re-sent ${file} lines ${repeated.slice(0, 5).join(',')}`);
|
||||||
|
}
|
||||||
|
}, 120_000);
|
||||||
|
|
||||||
|
it('spends the reclaimed bytes on source the agent has not seen', async () => {
|
||||||
|
const session = new ExploreSessionState();
|
||||||
|
const first = await explore(QUERY, session);
|
||||||
|
const second = await explore(QUERY, session);
|
||||||
|
|
||||||
|
const before = fencedLines(first);
|
||||||
|
const after = fencedLines(second);
|
||||||
|
const fresh = [...after.entries()].reduce(
|
||||||
|
(sum, [file, lines]) => sum + [...lines].filter((n) => !(before.get(file)?.has(n))).length, 0);
|
||||||
|
// Not merely "smaller": a shrunken response is what dedup must NOT produce.
|
||||||
|
// The freed budget has to come back as lines the first call never sent.
|
||||||
|
expect(fresh).toBeGreaterThan(20);
|
||||||
|
expect(second.length).toBeLessThan(first.length);
|
||||||
|
}, 120_000);
|
||||||
|
|
||||||
|
it('re-emits in full when the file changed between the two calls', async () => {
|
||||||
|
const session = new ExploreSessionState();
|
||||||
|
const target = path.join(testDir, 'internal/usecase/payroll/payslip_builder.go');
|
||||||
|
const original = fs.readFileSync(target, 'utf-8');
|
||||||
|
try {
|
||||||
|
const first = await explore(QUERY, session);
|
||||||
|
expect(fencedLines(first).has('internal/usecase/payroll/payslip_builder.go')).toBe(true);
|
||||||
|
|
||||||
|
fs.writeFileSync(target, original.replace('func sumKind(', 'func sumKindRenamed('), 'utf-8');
|
||||||
|
const second = await explore(QUERY, session);
|
||||||
|
|
||||||
|
// The edited file is served again, whole — a pointer here would send the
|
||||||
|
// agent to a copy of the file that no longer exists.
|
||||||
|
const pointerLines = second.split('\n').filter((l) => l.includes(POINTER));
|
||||||
|
expect(pointerLines.some((l) => l.includes('payslip_builder.go'))).toBe(false);
|
||||||
|
expect(fencedLines(second).get('internal/usecase/payroll/payslip_builder.go')?.size ?? 0)
|
||||||
|
.toBeGreaterThan(20);
|
||||||
|
} finally {
|
||||||
|
fs.writeFileSync(target, original, 'utf-8');
|
||||||
|
}
|
||||||
|
}, 120_000);
|
||||||
|
|
||||||
|
it('always returns real source, even when the session already holds everything', async () => {
|
||||||
|
const session = new ExploreSessionState();
|
||||||
|
await explore(QUERY, session);
|
||||||
|
await explore(QUERY, session);
|
||||||
|
const third = await explore(QUERY, session);
|
||||||
|
const fourth = await explore(QUERY, session);
|
||||||
|
|
||||||
|
// An all-pointer response is the shape that reads as failure. Every call
|
||||||
|
// keeps at least one real fenced block, however much the session holds.
|
||||||
|
for (const [n, text] of [[3, third], [4, fourth]] as const) {
|
||||||
|
const lines = [...fencedLines(text).values()].reduce((s, set) => s + set.size, 0);
|
||||||
|
expect(lines, `call ${n} returned no source at all`).toBeGreaterThan(10);
|
||||||
|
}
|
||||||
|
}, 180_000);
|
||||||
|
|
||||||
|
it('leaves the first call of a session untouched', async () => {
|
||||||
|
const tracked = await explore(QUERY, new ExploreSessionState());
|
||||||
|
const untracked = await explore(QUERY);
|
||||||
|
expect(tracked).toBe(untracked);
|
||||||
|
}, 120_000);
|
||||||
|
|
||||||
|
it('keeps two sessions on one handler independent', async () => {
|
||||||
|
const a = new ExploreSessionState();
|
||||||
|
const b = new ExploreSessionState();
|
||||||
|
const firstForA = await explore(QUERY, a);
|
||||||
|
await explore(QUERY, a);
|
||||||
|
// B's first call has seen nothing, whatever A has been served.
|
||||||
|
expect(await explore(QUERY, b)).toBe(firstForA);
|
||||||
|
}, 180_000);
|
||||||
|
|
||||||
|
it('is off entirely under CODEGRAPH_EXPLORE_DEDUP=0', async () => {
|
||||||
|
const session = new ExploreSessionState();
|
||||||
|
const previous = process.env.CODEGRAPH_EXPLORE_DEDUP;
|
||||||
|
process.env.CODEGRAPH_EXPLORE_DEDUP = '0';
|
||||||
|
try {
|
||||||
|
const first = await explore(QUERY, session);
|
||||||
|
const second = await explore(QUERY, session);
|
||||||
|
expect(second).toBe(first);
|
||||||
|
expect(second).not.toContain(POINTER);
|
||||||
|
} finally {
|
||||||
|
if (previous === undefined) delete process.env.CODEGRAPH_EXPLORE_DEDUP;
|
||||||
|
else process.env.CODEGRAPH_EXPLORE_DEDUP = previous;
|
||||||
|
}
|
||||||
|
}, 120_000);
|
||||||
|
|
||||||
|
it('reports the reclaimed bytes through the CG-4 diagnostic', async () => {
|
||||||
|
const sidecar = path.join(testDir, 'cg18-diagnostic.jsonl');
|
||||||
|
const session = new ExploreSessionState();
|
||||||
|
const previous = process.env.CODEGRAPH_EXPLORE_DEBUG;
|
||||||
|
process.env.CODEGRAPH_EXPLORE_DEBUG = sidecar;
|
||||||
|
try {
|
||||||
|
await explore(QUERY, session);
|
||||||
|
await explore(QUERY, session);
|
||||||
|
} finally {
|
||||||
|
if (previous === undefined) delete process.env.CODEGRAPH_EXPLORE_DEBUG;
|
||||||
|
else process.env.CODEGRAPH_EXPLORE_DEBUG = previous;
|
||||||
|
}
|
||||||
|
const [one, two] = fs.readFileSync(sidecar, 'utf-8').trim().split('\n').map((l) => JSON.parse(l));
|
||||||
|
|
||||||
|
expect(one.dedup.savedChars).toBe(0);
|
||||||
|
expect(two.dedup.savedChars).toBeGreaterThan(1000);
|
||||||
|
expect(two.dedup.backReferenced.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
// The reclamation is legible file by file: a back-referenced file spent
|
||||||
|
// none of its reservation, and the response still filled its envelope.
|
||||||
|
const backref = two.files.filter((f: { render: string }) => f.render === 'backref');
|
||||||
|
for (const f of backref) {
|
||||||
|
expect(f.emittedChars).toBe(0);
|
||||||
|
expect(f.dedupSavedChars).toBeGreaterThan(0);
|
||||||
|
expect(f.dedupCovered.length).toBeGreaterThan(0);
|
||||||
|
}
|
||||||
|
const spentOnFreshSource = two.files.reduce((s: number, f: { emittedChars: number }) => s + f.emittedChars, 0);
|
||||||
|
expect(spentOnFreshSource).toBeGreaterThan(0);
|
||||||
|
}, 120_000);
|
||||||
|
});
|
||||||
@@ -0,0 +1,249 @@
|
|||||||
|
/**
|
||||||
|
* Cross-call source dedup for `codegraph_explore` (CG-18).
|
||||||
|
*
|
||||||
|
* The session record (CG-17) knows what earlier calls already sent. This module
|
||||||
|
* is the algebra that turns that record into a decision for the call being
|
||||||
|
* rendered: of the line ranges this call WOULD emit, which does the agent
|
||||||
|
* already hold, and what is genuinely new.
|
||||||
|
*
|
||||||
|
* Three rules shape everything here, and all three come from the same place —
|
||||||
|
* an insufficient-feeling response is what sends an agent to Read, and one or
|
||||||
|
* two of those early in a session teach it to abandon codegraph entirely
|
||||||
|
* (CLAUDE.md):
|
||||||
|
*
|
||||||
|
* 1. **A pointer, never a bare omission.** Removed source is replaced by a
|
||||||
|
* back-reference naming the file, the symbols, and the line span, worded so
|
||||||
|
* it is unmistakable that the source was already delivered IN THIS
|
||||||
|
* CONVERSATION and is still current. Silence reads as "codegraph didn't
|
||||||
|
* find it".
|
||||||
|
* 2. **Only prove-it dedup.** A span is withheld only when the file's bytes
|
||||||
|
* are byte-identical to what was served (a content fingerprint, not an
|
||||||
|
* mtime and not the index's drift flag). An edit between calls means the
|
||||||
|
* agent's copy is wrong, so the source is re-emitted in full.
|
||||||
|
* 3. **Cut chunks, not slivers.** Only a covered run of at least
|
||||||
|
* {@link EXPLORE_DEDUP.MIN_COVERED_LINES} lines is worth replacing. Below
|
||||||
|
* that the pointer costs more than the source, and shattering a block into
|
||||||
|
* one-line fragments produces exactly the ragged output that reads as a
|
||||||
|
* failure. Everything not withheld is emitted — where the algebra is
|
||||||
|
* unsure, it re-serves.
|
||||||
|
*
|
||||||
|
* Which way to be wrong, restated for this layer: re-serving something the agent
|
||||||
|
* has is a few hundred wasted chars; withholding something it never saw is a
|
||||||
|
* Read. Every threshold below leans to the first.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { createHash } from 'crypto';
|
||||||
|
import type { ExploreLineRange, ExploreProjectState } from './explore-session-state';
|
||||||
|
|
||||||
|
export const EXPLORE_DEDUP = {
|
||||||
|
/**
|
||||||
|
* Shortest already-served run that may be replaced by a back-reference.
|
||||||
|
*
|
||||||
|
* Sized against what dedup is actually FOR — a later call re-serving a whole
|
||||||
|
* method or file it already sent. A shorter covered run is either a signature
|
||||||
|
* line in a skeleton render or the ±3 lines of context padding around a
|
||||||
|
* cluster, and swapping either for a pointer trades bytes for noise: the
|
||||||
|
* pointer sentence is itself ~140 chars, so under this length dedup would
|
||||||
|
* make the response BIGGER while making it read as full of holes.
|
||||||
|
*/
|
||||||
|
MIN_COVERED_LINES: 8,
|
||||||
|
/**
|
||||||
|
* Below this many chars of NEW source, a file's remainder is folded into its
|
||||||
|
* back-reference instead of being fenced on its own.
|
||||||
|
*
|
||||||
|
* The shape this exists for, seen on the CG-17 fixture: a third call whose
|
||||||
|
* only unheld line was the file's trailing blank one, rendered as a code fence
|
||||||
|
* containing `228\t`. A fence holding two lines of nothing reads as a broken
|
||||||
|
* response, and reading as broken is the expensive failure — it is the thing
|
||||||
|
* that sends an agent to Read and keeps it there. So a remainder this small is
|
||||||
|
* dropped rather than shown. It is the one place this module withholds
|
||||||
|
* something the agent has not seen, and it is bounded to ~two lines that sit
|
||||||
|
* directly against source the agent does hold; the file is still named, with
|
||||||
|
* its symbols, so one follow-up explore fetches it whole.
|
||||||
|
*/
|
||||||
|
MIN_DELTA_CHARS: 160,
|
||||||
|
/** Line spans named in one pointer before it summarises the rest. */
|
||||||
|
MAX_SPANS_IN_POINTER: 4,
|
||||||
|
/** Symbols named in one pointer before it summarises the rest. */
|
||||||
|
MAX_SYMBOLS_IN_POINTER: 5,
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
const OFF = new Set(['0', 'false', 'off', 'no']);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Kill switch: `CODEGRAPH_EXPLORE_DEDUP=0` renders every call as if the session
|
||||||
|
* had no history. Read per call (not memoized) so a test can toggle it.
|
||||||
|
*/
|
||||||
|
export function exploreDedupEnabled(): boolean {
|
||||||
|
const raw = process.env.CODEGRAPH_EXPLORE_DEDUP;
|
||||||
|
if (raw === undefined) return true;
|
||||||
|
return !OFF.has(raw.trim().toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Identity of the bytes a call served for one file.
|
||||||
|
*
|
||||||
|
* This — not the index's drift flag — is what gates dedup. `isFileStaleOnDisk`
|
||||||
|
* answers "did the file change since the last INDEX SYNC", which is a different
|
||||||
|
* question with a different answer: two calls inside one drift window served the
|
||||||
|
* same current bytes (dedup is correct), while a file edited and re-synced
|
||||||
|
* between two calls is never "stale" and yet the agent's copy is now wrong
|
||||||
|
* (dedup would be actively harmful). Length is prefixed so a hash prefix
|
||||||
|
* collision cannot alias two files of different size.
|
||||||
|
*/
|
||||||
|
export function fileFingerprint(content: string): string {
|
||||||
|
return `${content.length}:${createHash('sha1').update(content).digest('hex').slice(0, 16)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Sort + merge overlapping/adjacent spans into the smallest equivalent set. */
|
||||||
|
export function mergeRanges(ranges: ReadonlyArray<ExploreLineRange>): ExploreLineRange[] {
|
||||||
|
const valid = ranges
|
||||||
|
.filter((r) => Number.isFinite(r.start) && Number.isFinite(r.end) && r.end >= r.start && r.start >= 1)
|
||||||
|
.map((r) => ({ start: Math.floor(r.start), end: Math.floor(r.end) }))
|
||||||
|
.sort((a, b) => a.start - b.start || a.end - b.end);
|
||||||
|
const out: ExploreLineRange[] = [];
|
||||||
|
for (const r of valid) {
|
||||||
|
const last = out[out.length - 1];
|
||||||
|
if (last && r.start <= last.end + 1) last.end = Math.max(last.end, r.end);
|
||||||
|
else out.push({ ...r });
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The parts of `range` that `served` covers. */
|
||||||
|
export function intersectRange(
|
||||||
|
range: ExploreLineRange,
|
||||||
|
served: ReadonlyArray<ExploreLineRange>,
|
||||||
|
): ExploreLineRange[] {
|
||||||
|
const out: ExploreLineRange[] = [];
|
||||||
|
for (const s of served) {
|
||||||
|
const start = Math.max(range.start, s.start);
|
||||||
|
const end = Math.min(range.end, s.end);
|
||||||
|
if (end >= start) out.push({ start, end });
|
||||||
|
}
|
||||||
|
return mergeRanges(out);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The parts of `range` that `cut` does NOT cover. */
|
||||||
|
export function subtractRange(
|
||||||
|
range: ExploreLineRange,
|
||||||
|
cut: ReadonlyArray<ExploreLineRange>,
|
||||||
|
): ExploreLineRange[] {
|
||||||
|
const out: ExploreLineRange[] = [];
|
||||||
|
let cursor = range.start;
|
||||||
|
for (const c of mergeRanges(cut)) {
|
||||||
|
if (c.end < cursor) continue;
|
||||||
|
if (c.start > range.end) break;
|
||||||
|
if (c.start > cursor) out.push({ start: cursor, end: Math.min(c.start - 1, range.end) });
|
||||||
|
cursor = Math.max(cursor, c.end + 1);
|
||||||
|
if (cursor > range.end) break;
|
||||||
|
}
|
||||||
|
if (cursor <= range.end) out.push({ start: cursor, end: range.end });
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What one intended span becomes once the session's history is applied. */
|
||||||
|
export interface RangeDedup {
|
||||||
|
/** Spans to render now — everything not proven-already-held. */
|
||||||
|
emit: ExploreLineRange[];
|
||||||
|
/** Spans replaced by a back-reference. */
|
||||||
|
covered: ExploreLineRange[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Split one intended span into what to emit and what to point back at.
|
||||||
|
*
|
||||||
|
* Covered runs shorter than {@link EXPLORE_DEDUP.MIN_COVERED_LINES} are left in
|
||||||
|
* the emit set on purpose (rule 3 above) — so a span the agent holds "almost
|
||||||
|
* all of" still comes back whole rather than as a stutter of fragments around
|
||||||
|
* pointers.
|
||||||
|
*/
|
||||||
|
export function dedupeRange(
|
||||||
|
range: ExploreLineRange,
|
||||||
|
served: ReadonlyArray<ExploreLineRange>,
|
||||||
|
minCovered: number = EXPLORE_DEDUP.MIN_COVERED_LINES,
|
||||||
|
): RangeDedup {
|
||||||
|
if (served.length === 0 || range.end < range.start) return { emit: [range], covered: [] };
|
||||||
|
const covered = intersectRange(range, served).filter((r) => r.end - r.start + 1 >= minCovered);
|
||||||
|
if (covered.length === 0) return { emit: [range], covered: [] };
|
||||||
|
return { emit: subtractRange(range, covered), covered };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every line span this session has already served for one file, but ONLY from
|
||||||
|
* calls that served the SAME BYTES.
|
||||||
|
*
|
||||||
|
* A record with no fingerprint is ignored rather than trusted: it cannot prove
|
||||||
|
* the agent's copy matches the file on disk now, and an unprovable match is
|
||||||
|
* exactly the case where re-serving is right.
|
||||||
|
*/
|
||||||
|
export function servedRangesForFile(
|
||||||
|
prior: ExploreProjectState | null,
|
||||||
|
filePath: string,
|
||||||
|
fingerprint: string,
|
||||||
|
): ExploreLineRange[] {
|
||||||
|
if (!prior) return [];
|
||||||
|
const spans: ExploreLineRange[] = [];
|
||||||
|
for (const call of prior.calls) {
|
||||||
|
for (const file of call.files) {
|
||||||
|
if (file.path !== filePath) continue;
|
||||||
|
if (!file.fingerprint || file.fingerprint !== fingerprint) continue;
|
||||||
|
spans.push(...file.ranges);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return mergeRanges(spans);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `L12`, `L12-40`, capped with a `+N more` tail. */
|
||||||
|
export function formatSpans(spans: ReadonlyArray<ExploreLineRange>): string {
|
||||||
|
const shown = spans.slice(0, EXPLORE_DEDUP.MAX_SPANS_IN_POINTER)
|
||||||
|
.map((r) => (r.start === r.end ? `L${r.start}` : `L${r.start}-${r.end}`))
|
||||||
|
.join(', ');
|
||||||
|
const more = spans.length - EXPLORE_DEDUP.MAX_SPANS_IN_POINTER;
|
||||||
|
return more > 0 ? `${shown}, +${more} more span${more === 1 ? '' : 's'}` : shown;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The line that replaces withheld source.
|
||||||
|
*
|
||||||
|
* It has one job: make the agent reach into its own context instead of into
|
||||||
|
* Read. So it carries the three things needed to find the source it already has
|
||||||
|
* — path, symbols, line spans — plus the two facts that make using it safe:
|
||||||
|
* that it came from THIS conversation, and that the file has not changed since
|
||||||
|
* (which is checked, not asserted — see {@link fileFingerprint}). It never says
|
||||||
|
* "omitted", and it never steers to Read.
|
||||||
|
*/
|
||||||
|
export function formatBackReference(
|
||||||
|
filePath: string,
|
||||||
|
covered: ReadonlyArray<ExploreLineRange>,
|
||||||
|
symbols: ReadonlyArray<string>,
|
||||||
|
opts: { partial: boolean },
|
||||||
|
): string {
|
||||||
|
const names = symbols.slice(0, EXPLORE_DEDUP.MAX_SYMBOLS_IN_POINTER);
|
||||||
|
const moreNames = symbols.length - names.length;
|
||||||
|
const symbolPart = names.length > 0
|
||||||
|
? ` (${names.join(', ')}${moreNames > 0 ? `, +${moreNames} more` : ''})`
|
||||||
|
: '';
|
||||||
|
const head = `> **Already sent earlier in this conversation:** \`${filePath}\` ${formatSpans(covered)}${symbolPart}`;
|
||||||
|
const tail = opts.partial
|
||||||
|
? ' — unchanged on disk since, so that copy is still exact. Only the NEW lines are shown below; scroll back for the rest. Do NOT Read this file.'
|
||||||
|
: ' — unchanged on disk since, so that copy is still exact and is not repeated here. Use it from your context; do NOT Read this file.';
|
||||||
|
return head + tail;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Symbol names whose definitions fall inside the withheld spans. */
|
||||||
|
export function symbolsInSpans(
|
||||||
|
nodes: ReadonlyArray<{ name: string; kind: string; startLine: number; endLine: number }>,
|
||||||
|
spans: ReadonlyArray<ExploreLineRange>,
|
||||||
|
): string[] {
|
||||||
|
const out: string[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
for (const n of nodes) {
|
||||||
|
if (n.kind === 'import' || n.kind === 'export') continue;
|
||||||
|
if (!spans.some((s) => n.startLine <= s.end && (n.endLine || n.startLine) >= s.start)) continue;
|
||||||
|
if (seen.has(n.name)) continue;
|
||||||
|
seen.add(n.name);
|
||||||
|
out.push(n.name);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
@@ -41,6 +41,7 @@ export type ExploreRenderMode =
|
|||||||
| 'focused' // per-symbol view, named/spine bodies full
|
| 'focused' // per-symbol view, named/spine bodies full
|
||||||
| 'skeleton' // per-symbol view, signatures only
|
| 'skeleton' // per-symbol view, signatures only
|
||||||
| 'stale-omitted' // drifted on disk; source deliberately withheld
|
| 'stale-omitted' // drifted on disk; source deliberately withheld
|
||||||
|
| 'backref' // fully served by an earlier call this session (CG-18)
|
||||||
| 'dropped'; // rendered into `lines` but cut by the final hard ceiling
|
| 'dropped'; // rendered into `lines` but cut by the final hard ceiling
|
||||||
|
|
||||||
/** Why a ranked candidate never reached the output. */
|
/** Why a ranked candidate never reached the output. */
|
||||||
@@ -92,6 +93,17 @@ interface FileRecord extends ExploreCandidateMeta {
|
|||||||
*/
|
*/
|
||||||
allowance: number | null;
|
allowance: number | null;
|
||||||
render?: ExploreRenderMode;
|
render?: ExploreRenderMode;
|
||||||
|
/**
|
||||||
|
* Source chars this call did NOT re-send because an earlier call in the
|
||||||
|
* session already did (CG-18). Reclaimed, not lost: it leaves through
|
||||||
|
* `sourceSpent` (the carry-forward pool hands it to lower-ranked files) and,
|
||||||
|
* for a fully back-referenced file, through the freed `maxFiles` slot. The
|
||||||
|
* reallocation is legible as the difference between this file's
|
||||||
|
* `allowance` and `emittedChars` against the files below it in the table.
|
||||||
|
*/
|
||||||
|
dedupSavedChars: number;
|
||||||
|
/** Line spans replaced by a back-reference. */
|
||||||
|
dedupCovered: Array<[number, number]>;
|
||||||
/** Source chars the render loop handed to `lines` (pre-final-truncation). */
|
/** Source chars the render loop handed to `lines` (pre-final-truncation). */
|
||||||
emittedChars: number;
|
emittedChars: number;
|
||||||
/** Source chars present in the FINAL text — authoritative, truncation-aware. */
|
/** Source chars present in the FINAL text — authoritative, truncation-aware. */
|
||||||
@@ -130,6 +142,8 @@ export interface ExploreDiagnosticFile extends ExploreCandidateMeta {
|
|||||||
render: ExploreRenderMode | null;
|
render: ExploreRenderMode | null;
|
||||||
skipped: ExploreSkipReason | null;
|
skipped: ExploreSkipReason | null;
|
||||||
clipped: boolean;
|
clipped: boolean;
|
||||||
|
dedupSavedChars: number;
|
||||||
|
dedupCovered: Array<[number, number]>;
|
||||||
emittedChars: number;
|
emittedChars: number;
|
||||||
finalChars: number;
|
finalChars: number;
|
||||||
share: number;
|
share: number;
|
||||||
@@ -191,6 +205,17 @@ export interface ExploreDiagnosticReport {
|
|||||||
filesRenderedByLoop: number;
|
filesRenderedByLoop: number;
|
||||||
filesInFinalOutput: number;
|
filesInFinalOutput: number;
|
||||||
};
|
};
|
||||||
|
/**
|
||||||
|
* Cross-call source dedup (CG-18): what this call did NOT re-send because an
|
||||||
|
* earlier call in this session already sent it, and where those bytes went.
|
||||||
|
* `savedChars` 0 with a non-empty session block means nothing overlapped.
|
||||||
|
*/
|
||||||
|
dedup: {
|
||||||
|
savedChars: number;
|
||||||
|
backReferenced: string[];
|
||||||
|
/** Files fully replaced by a pointer — each one also freed a `maxFiles` slot. */
|
||||||
|
fullyBackReferenced: string[];
|
||||||
|
};
|
||||||
/** The proportional split (CG-12): what each file was promised, and why. */
|
/** The proportional split (CG-12): what each file was promised, and why. */
|
||||||
allocation: {
|
allocation: {
|
||||||
/** Chars divided among admitted files (envelope minus per-file overhead). */
|
/** Chars divided among admitted files (envelope minus per-file overhead). */
|
||||||
@@ -338,6 +363,7 @@ export class ExploreDiagnostics {
|
|||||||
noteCandidate(path: string, meta: ExploreCandidateMeta): void {
|
noteCandidate(path: string, meta: ExploreCandidateMeta): void {
|
||||||
this.files.set(path, {
|
this.files.set(path, {
|
||||||
path, ...meta, allowance: null,
|
path, ...meta, allowance: null,
|
||||||
|
dedupSavedChars: 0, dedupCovered: [],
|
||||||
emittedChars: 0, finalChars: 0, share: 0, allocatedShare: 0, clipped: false,
|
emittedChars: 0, finalChars: 0, share: 0, allocatedShare: 0, clipped: false,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -375,6 +401,19 @@ export class ExploreDiagnostics {
|
|||||||
rec.skipped = undefined;
|
rec.skipped = undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Source this call withheld because the session already holds it (CG-18).
|
||||||
|
* Called with `(path, 0, [])` to clear a record — the anti-abandonment restore
|
||||||
|
* puts a suppressed file's source back, and a diagnostic still claiming the
|
||||||
|
* saving would misreport where the envelope went.
|
||||||
|
*/
|
||||||
|
recordDedup(path: string, savedChars: number, covered: ReadonlyArray<{ start: number; end: number }>): void {
|
||||||
|
const rec = this.files.get(path);
|
||||||
|
if (!rec) return;
|
||||||
|
rec.dedupSavedChars = savedChars;
|
||||||
|
rec.dedupCovered = covered.map((r) => [r.start, r.end] as [number, number]);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A candidate was passed over before rendering. First reason wins — the
|
* A candidate was passed over before rendering. First reason wins — the
|
||||||
* blanket `max-files` sweep must not overwrite a file's specific reason.
|
* blanket `max-files` sweep must not overwrite a file's specific reason.
|
||||||
@@ -410,8 +449,10 @@ export class ExploreDiagnostics {
|
|||||||
rec.share = envelope > 0 ? rec.finalChars / envelope : 0;
|
rec.share = envelope > 0 ? rec.finalChars / envelope : 0;
|
||||||
rec.allocatedShare = allocatedChars > 0 ? rec.emittedChars / allocatedChars : 0;
|
rec.allocatedShare = allocatedChars > 0 ? rec.emittedChars / allocatedChars : 0;
|
||||||
// Rendered into `lines` but absent from the final text → the hard
|
// Rendered into `lines` but absent from the final text → the hard
|
||||||
// ceiling dropped its whole section.
|
// ceiling dropped its whole section. A back-referenced file has no
|
||||||
if (rec.render && rec.render !== 'stale-omitted' && rec.finalChars === 0) {
|
// fenced source BY DESIGN (CG-18), so it is never "dropped".
|
||||||
|
if (rec.render && rec.render !== 'stale-omitted' && rec.render !== 'backref'
|
||||||
|
&& rec.finalChars === 0) {
|
||||||
rec.render = 'dropped';
|
rec.render = 'dropped';
|
||||||
rec.clipped = true;
|
rec.clipped = true;
|
||||||
}
|
}
|
||||||
@@ -467,6 +508,11 @@ export class ExploreDiagnostics {
|
|||||||
filesRenderedByLoop: filesIncluded,
|
filesRenderedByLoop: filesIncluded,
|
||||||
filesInFinalOutput: rendered.length,
|
filesInFinalOutput: rendered.length,
|
||||||
},
|
},
|
||||||
|
dedup: {
|
||||||
|
savedChars: records.reduce((s, r) => s + r.dedupSavedChars, 0),
|
||||||
|
backReferenced: records.filter((r) => r.dedupSavedChars > 0).map((r) => r.path),
|
||||||
|
fullyBackReferenced: records.filter((r) => r.render === 'backref').map((r) => r.path),
|
||||||
|
},
|
||||||
allocation: {
|
allocation: {
|
||||||
pool: this.allocPool,
|
pool: this.allocPool,
|
||||||
cliffAt: round6(this.allocCliffAt),
|
cliffAt: round6(this.allocCliffAt),
|
||||||
@@ -495,6 +541,8 @@ export class ExploreDiagnostics {
|
|||||||
render: r.render ?? null,
|
render: r.render ?? null,
|
||||||
skipped: r.skipped ?? null,
|
skipped: r.skipped ?? null,
|
||||||
clipped: r.clipped,
|
clipped: r.clipped,
|
||||||
|
dedupSavedChars: r.dedupSavedChars,
|
||||||
|
dedupCovered: r.dedupCovered.map((s) => [...s] as [number, number]),
|
||||||
emittedChars: r.emittedChars,
|
emittedChars: r.emittedChars,
|
||||||
finalChars: r.finalChars,
|
finalChars: r.finalChars,
|
||||||
share: round6(r.share),
|
share: round6(r.share),
|
||||||
@@ -614,6 +662,15 @@ export function renderTable(report: ExploreDiagnosticReport): string {
|
|||||||
` relevance gate ${sel.graphGateApplied ? 'applied' : 'not applied'}` +
|
` relevance gate ${sel.graphGateApplied ? 'applied' : 'not applied'}` +
|
||||||
` at graph >= ${sel.graphGateThreshold.toFixed(5)} (6% of max ${sel.maxGraph.toFixed(5)})`,
|
` at graph >= ${sel.graphGateThreshold.toFixed(5)} (6% of max ${sel.maxGraph.toFixed(5)})`,
|
||||||
);
|
);
|
||||||
|
const dedup = report.dedup;
|
||||||
|
if (dedup && dedup.savedChars > 0) {
|
||||||
|
out.push(
|
||||||
|
` dedup ${num(dedup.savedChars)} chars not re-sent` +
|
||||||
|
` · ${dedup.fullyBackReferenced.length} file(s) fully back-referenced` +
|
||||||
|
` (each also freed a maxFiles slot)` +
|
||||||
|
(dedup.backReferenced.length > 0 ? `: ${dedup.backReferenced.join(', ')}` : ''),
|
||||||
|
);
|
||||||
|
}
|
||||||
const alloc = report.allocation;
|
const alloc = report.allocation;
|
||||||
out.push(
|
out.push(
|
||||||
` allocation ${num(alloc.reserved)} reserved of ${num(alloc.pool)} pool` +
|
` allocation ${num(alloc.reserved)} reserved of ${num(alloc.pool)} pool` +
|
||||||
@@ -627,7 +684,7 @@ export function renderTable(report: ExploreDiagnosticReport): string {
|
|||||||
// Allocated (not delivered) is the allocator's own decision — the number the
|
// Allocated (not delivered) is the allocator's own decision — the number the
|
||||||
// budget work is about. Delivered is what the agent got. They differ only
|
// budget work is about. Delivered is what the agent got. They differ only
|
||||||
// when the ceiling truncated; showing both makes that divergence obvious.
|
// when the ceiling truncated; showing both makes that divergence obvious.
|
||||||
const shown = files.filter((f) => f.emittedChars > 0 || f.finalChars > 0);
|
const shown = files.filter((f) => f.emittedChars > 0 || f.finalChars > 0 || f.render === 'backref');
|
||||||
if (shown.length > 0) {
|
if (shown.length > 0) {
|
||||||
out.push(' # alloc% deliv% bytes reserved score graph hits pen flags render file');
|
out.push(' # alloc% deliv% bytes reserved score graph hits pen flags render file');
|
||||||
for (const f of shown) {
|
for (const f of shown) {
|
||||||
@@ -647,6 +704,11 @@ export function renderTable(report: ExploreDiagnosticReport): string {
|
|||||||
f.path,
|
f.path,
|
||||||
);
|
);
|
||||||
out.push(' kinds: ' + (f.kinds || '-'));
|
out.push(' kinds: ' + (f.kinds || '-'));
|
||||||
|
if (f.dedupSavedChars > 0) {
|
||||||
|
const spans = f.dedupCovered.slice(0, 6).map(([a, b]) => (a === b ? `${a}` : `${a}-${b}`)).join(',');
|
||||||
|
const more = f.dedupCovered.length > 6 ? `,+${f.dedupCovered.length - 6}` : '';
|
||||||
|
out.push(` dedup: ${num(f.dedupSavedChars)} chars already sent this session · L${spans}${more}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
out.push(' (bytes = source allocated by the render loop; deliv% = 0 means the hard ceiling dropped the section)');
|
out.push(' (bytes = source allocated by the render loop; deliv% = 0 means the hard ceiling dropped the section)');
|
||||||
out.push(' (* = clipped: some source in this file was elided, windowed, or its section dropped)');
|
out.push(' (* = clipped: some source in this file was elided, windowed, or its section dropped)');
|
||||||
|
|||||||
@@ -70,6 +70,13 @@ export interface ExploreFileEmission {
|
|||||||
ranges: ExploreLineRange[];
|
ranges: ExploreLineRange[];
|
||||||
/** Source chars emitted for this file (excludes headers / fences). */
|
/** Source chars emitted for this file (excludes headers / fences). */
|
||||||
bytes: number;
|
bytes: number;
|
||||||
|
/**
|
||||||
|
* Identity of the bytes those ranges were sliced from (CG-18). Cross-call
|
||||||
|
* dedup withholds a span only when the file still hashes to this, so an edit
|
||||||
|
* between two calls re-serves instead of pointing at source the agent holds a
|
||||||
|
* now-wrong copy of. Absent = unprovable, which dedup treats as "re-serve".
|
||||||
|
*/
|
||||||
|
fingerprint?: string;
|
||||||
/** Set when ranges were dropped to stay under the per-file bound. */
|
/** Set when ranges were dropped to stay under the per-file bound. */
|
||||||
rangesTruncated?: boolean;
|
rangesTruncated?: boolean;
|
||||||
}
|
}
|
||||||
@@ -309,6 +316,7 @@ export class ExploreSessionState {
|
|||||||
.map((f) => {
|
.map((f) => {
|
||||||
const { ranges, truncated } = coalesceRanges(f.ranges ?? []);
|
const { ranges, truncated } = coalesceRanges(f.ranges ?? []);
|
||||||
const out: ExploreFileEmission = { path: f.path, ranges, bytes: Math.max(0, f.bytes || 0) };
|
const out: ExploreFileEmission = { path: f.path, ranges, bytes: Math.max(0, f.bytes || 0) };
|
||||||
|
if (typeof f.fingerprint === 'string' && f.fingerprint) out.fingerprint = f.fingerprint;
|
||||||
if (truncated) out.rangesTruncated = true;
|
if (truncated) out.rangesTruncated = true;
|
||||||
return out;
|
return out;
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -62,6 +62,8 @@ calls; a grep/read exploration is dozens.
|
|||||||
- **After editing, check the staleness banner.** When a tool response starts with "⚠️ Some files referenced below were edited since the last index sync…", the listed files are pending re-index — Read those specific files for accurate content. Every file NOT in that banner is fresh, so still trust codegraph. A different, rarer banner — "⚠️ CodeGraph auto-sync is DISABLED…" — means live watching stopped entirely (the whole index is frozen, not just a few files); until it's resolved, Read files directly to confirm anything that may have changed.
|
- **After editing, check the staleness banner.** When a tool response starts with "⚠️ Some files referenced below were edited since the last index sync…", the listed files are pending re-index — Read those specific files for accurate content. Every file NOT in that banner is fresh, so still trust codegraph. A different, rarer banner — "⚠️ CodeGraph auto-sync is DISABLED…" — means live watching stopped entirely (the whole index is frozen, not just a few files); until it's resolved, Read files directly to confirm anything that may have changed.
|
||||||
- **A file flagged "⚠ changed on disk after the last index sync" drifted from its index** (most common on projects queried via \`projectPath\`, which have no live watcher). Codegraph never serves a possibly-mis-sliced body from such a file — it either shows the file's full CURRENT source (trust it as a Read) or omits the source with this flag. When the source was omitted, Read that specific file; line numbers referencing it elsewhere in the response may be shifted until that project's next sync. All unflagged files remain trustworthy.
|
- **A file flagged "⚠ changed on disk after the last index sync" drifted from its index** (most common on projects queried via \`projectPath\`, which have no live watcher). Codegraph never serves a possibly-mis-sliced body from such a file — it either shows the file's full CURRENT source (trust it as a Read) or omits the source with this flag. When the source was omitted, Read that specific file; line numbers referencing it elsewhere in the response may be shifted until that project's next sync. All unflagged files remain trustworthy.
|
||||||
|
|
||||||
|
- **"Already sent earlier in this conversation" is a pointer, not a gap.** When a file's section carries that line instead of (or above) its source, an earlier \`codegraph_explore\` in THIS conversation already returned those exact lines and the file has not changed since — so the copy already in your context is current and exact. Scroll back to it; don't re-fetch it and don't Read the file. The bytes it freed went into source you have not seen yet, elsewhere in the same response.
|
||||||
|
|
||||||
## Limitations
|
## Limitations
|
||||||
|
|
||||||
- If a tool reports a project isn't indexed (no \`.codegraph/\`), stop calling codegraph tools for that project for the rest of the session and use your built-in tools there instead. Indexing is the user's decision — mention they can run \`codegraph init\` if it comes up, but don't run it yourself.
|
- If a tool reports a project isn't indexed (no \`.codegraph/\`), stop calling codegraph tools for that project for the rest of the session and use your built-in tools there instead. Indexing is the user's decision — mention they can run \`codegraph init\` if it comes up, but don't run it yourself.
|
||||||
|
|||||||
+383
-89
@@ -52,6 +52,16 @@ import {
|
|||||||
type ExploreFileEmission,
|
type ExploreFileEmission,
|
||||||
type ExploreLineRange,
|
type ExploreLineRange,
|
||||||
} from './explore-session-state';
|
} from './explore-session-state';
|
||||||
|
import {
|
||||||
|
EXPLORE_DEDUP,
|
||||||
|
dedupeRange,
|
||||||
|
exploreDedupEnabled,
|
||||||
|
fileFingerprint,
|
||||||
|
formatBackReference,
|
||||||
|
mergeRanges,
|
||||||
|
servedRangesForFile,
|
||||||
|
symbolsInSpans,
|
||||||
|
} from './explore-dedup';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An expected, recoverable "codegraph can't serve this" condition — most
|
* An expected, recoverable "codegraph can't serve this" condition — most
|
||||||
@@ -3120,26 +3130,50 @@ export class ToolHandler {
|
|||||||
// byte-identical. It only OBSERVES: it must never feed back into rendering.
|
// byte-identical. It only OBSERVES: it must never feed back into rendering.
|
||||||
const diag = ExploreDiagnostics.start(query, projectRoot, budget, maxFiles, indexedFileCount);
|
const diag = ExploreDiagnostics.start(query, projectRoot, budget, maxFiles, indexedFileCount);
|
||||||
|
|
||||||
// What this session has already been served for THIS project (CG-17).
|
// What this session has already been served for THIS project (CG-17), and
|
||||||
// Read-only at this stage — it is reported in the diagnostic and nothing
|
// whether this call may act on it (CG-18). Dedup is off on the session's
|
||||||
// else, so the response is unchanged. Cross-call dedup (CG-18) and budget
|
// first call by construction — there is nothing to point back AT — and off
|
||||||
// decay (CG-19) are the consumers this exists for.
|
// entirely under `CODEGRAPH_EXPLORE_DEDUP=0`.
|
||||||
const priorCalls = viewForProject(readExploreSessionView(args), projectRoot);
|
const priorCalls = viewForProject(readExploreSessionView(args), projectRoot);
|
||||||
diag?.noteSession(priorCalls);
|
diag?.noteSession(priorCalls);
|
||||||
|
const dedupEnabled = exploreDedupEnabled() && (priorCalls?.calls.length ?? 0) > 0;
|
||||||
|
|
||||||
|
// Cross-call dedup accounting (CG-18). `newSourceChars` is the load-bearing
|
||||||
|
// one: a response whose source is ENTIRELY back-references is the shape that
|
||||||
|
// reads as a failure, so the loop keeps the top suppressed file's real
|
||||||
|
// section in hand and restores it if nothing new made it in — see
|
||||||
|
// `suppressedFallback` below.
|
||||||
|
let newSourceChars = 0;
|
||||||
|
const backReferencedFiles: string[] = [];
|
||||||
|
|
||||||
// What this call ends up emitting, per file — the record handed back to the
|
// What this call ends up emitting, per file — the record handed back to the
|
||||||
// session state on the main thread. Filled by every render path below, then
|
// session state on the main thread. Filled by every render path below, then
|
||||||
// filtered to the files that SURVIVE the final hard-ceiling cut, so the
|
// filtered to the files that SURVIVE the final hard-ceiling cut, so the
|
||||||
// record is what the agent actually received rather than what the loop
|
// record is what the agent actually received rather than what the loop
|
||||||
// hoped to send.
|
// hoped to send.
|
||||||
const emittedByFile = new Map<string, { ranges: ExploreLineRange[]; bytes: number }>();
|
//
|
||||||
const noteEmitted = (fp: string, ranges: ExploreLineRange[], bytes: number): void => {
|
// Back-referenced spans are recorded too, with zero bytes (CG-18): the
|
||||||
|
// record means "source the agent HOLDS for this file", not "bytes this call
|
||||||
|
// spent". Re-recording them refreshes them inside the retained-call window,
|
||||||
|
// so a file pointed at across many calls doesn't age out of the history and
|
||||||
|
// get re-served for no reason.
|
||||||
|
const emittedByFile = new Map<
|
||||||
|
string,
|
||||||
|
{ ranges: ExploreLineRange[]; bytes: number; fingerprint?: string }
|
||||||
|
>();
|
||||||
|
const noteEmitted = (
|
||||||
|
fp: string,
|
||||||
|
ranges: ExploreLineRange[],
|
||||||
|
bytes: number,
|
||||||
|
fingerprint?: string,
|
||||||
|
): void => {
|
||||||
const existing = emittedByFile.get(fp);
|
const existing = emittedByFile.get(fp);
|
||||||
if (existing) {
|
if (existing) {
|
||||||
existing.ranges.push(...ranges);
|
existing.ranges.push(...ranges);
|
||||||
existing.bytes += bytes;
|
existing.bytes += bytes;
|
||||||
|
if (fingerprint) existing.fingerprint = fingerprint;
|
||||||
} else {
|
} else {
|
||||||
emittedByFile.set(fp, { ranges: [...ranges], bytes });
|
emittedByFile.set(fp, { ranges: [...ranges], bytes, fingerprint });
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -3976,6 +4010,28 @@ export class ToolHandler {
|
|||||||
// instead of a different symbol's code under the requested name.
|
// instead of a different symbol's code under the requested name.
|
||||||
const staleRendered: string[] = [];
|
const staleRendered: string[] = [];
|
||||||
const staleOmitted: string[] = [];
|
const staleOmitted: string[] = [];
|
||||||
|
// Anti-abandonment hold-back (CG-18). The first file dedup suppressed
|
||||||
|
// ENTIRELY, kept with its real section so it can be put back if the loop
|
||||||
|
// ends with no new source anywhere. A response made only of pointers is the
|
||||||
|
// shape that reads as "codegraph has nothing" — and one such response early
|
||||||
|
// in a session is enough to make an agent stop calling the tool at all — so
|
||||||
|
// the highest-ranked suppressed file is restored rather than risk it. It
|
||||||
|
// costs a re-serve of one file, on the one call shape where dedup would
|
||||||
|
// otherwise have saved everything: the safe direction.
|
||||||
|
type SuppressedFallback = {
|
||||||
|
filePath: string;
|
||||||
|
/** Index in `lines` where this file's pointer block starts. */
|
||||||
|
at: number;
|
||||||
|
/** How many `lines` entries the pointer block occupies. */
|
||||||
|
replacing: number;
|
||||||
|
/** The full, undeduped section to splice back in. */
|
||||||
|
section: string[];
|
||||||
|
sourceChars: number;
|
||||||
|
overhead: number;
|
||||||
|
ranges: ExploreLineRange[];
|
||||||
|
fingerprint: string;
|
||||||
|
};
|
||||||
|
let suppressedFallback: SuppressedFallback | null = null;
|
||||||
// Reservation carry-forward (CG-21). A reservation is a promise the render
|
// Reservation carry-forward (CG-21). A reservation is a promise the render
|
||||||
// loop has to KEEP, not a cap it may quietly under-use: a file that cannot
|
// loop has to KEEP, not a cap it may quietly under-use: a file that cannot
|
||||||
// spend what it was given — thin matched-symbol set, unreadable, drifted off
|
// spend what it was given — thin matched-symbol set, unreadable, drifted off
|
||||||
@@ -4053,6 +4109,135 @@ export class ToolHandler {
|
|||||||
|
|
||||||
const fileLines = fileContent.split('\n');
|
const fileLines = fileContent.split('\n');
|
||||||
const lang = group.nodes[0]?.language || '';
|
const lang = group.nodes[0]?.language || '';
|
||||||
|
const withLineNumbers = exploreLineNumbersEnabled();
|
||||||
|
// Language-neutral separator between two non-contiguous slices of one file
|
||||||
|
// (no `//` — not a comment in Python, Ruby, etc.). With line numbers on,
|
||||||
|
// the line-number jump also signals the gap.
|
||||||
|
const GAP_MARKER = '\n\n... (gap) ...\n\n';
|
||||||
|
|
||||||
|
// Cross-call dedup (CG-18). `served` is what THIS session already sent the
|
||||||
|
// agent for THIS file, and it is empty unless the file still hashes to the
|
||||||
|
// bytes those spans were sliced from — an edit between calls means the
|
||||||
|
// agent's copy is wrong, so nothing is withheld. Every render path below
|
||||||
|
// routes its spans through `dedupeSpans`, which is the only place a span
|
||||||
|
// is ever dropped.
|
||||||
|
const fingerprint = fileFingerprint(fileContent);
|
||||||
|
const served = dedupEnabled ? servedRangesForFile(priorCalls, filePath, fingerprint) : [];
|
||||||
|
/** Render one line span exactly as the render paths do. */
|
||||||
|
const renderSpan = (r: ExploreLineRange): string => {
|
||||||
|
const slice = fileLines.slice(r.start - 1, r.end).join('\n');
|
||||||
|
return withLineNumbers ? numberSourceLines(slice, r.start) : slice;
|
||||||
|
};
|
||||||
|
/**
|
||||||
|
* Apply the session history to a set of spans-with-text. A span the agent
|
||||||
|
* already holds is dropped and reported in `covered`; a partially-held one
|
||||||
|
* is re-rendered down to its new lines. Text is rebuilt from the surviving
|
||||||
|
* spans rather than sliced out of the original string — the spans ARE the
|
||||||
|
* contract with the session record, so rebuilding from them is what keeps
|
||||||
|
* what we claim to have sent and what we sent the same thing.
|
||||||
|
*/
|
||||||
|
const dedupeSpans = (
|
||||||
|
parts: ReadonlyArray<{ range: ExploreLineRange; text: string }>,
|
||||||
|
): { parts: Array<{ range: ExploreLineRange; text: string }>; covered: ExploreLineRange[] } => {
|
||||||
|
if (served.length === 0) return { parts: [...parts], covered: [] };
|
||||||
|
const kept: Array<{ range: ExploreLineRange; text: string }> = [];
|
||||||
|
const covered: ExploreLineRange[] = [];
|
||||||
|
for (const part of parts) {
|
||||||
|
const split = dedupeRange(part.range, served);
|
||||||
|
if (split.covered.length === 0) {
|
||||||
|
kept.push(part);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
covered.push(...split.covered);
|
||||||
|
for (const r of split.emit) kept.push({ range: r, text: renderSpan(r) });
|
||||||
|
}
|
||||||
|
return { parts: kept, covered: mergeRanges(covered) };
|
||||||
|
};
|
||||||
|
const coveredChars = (spans: ReadonlyArray<ExploreLineRange>): number =>
|
||||||
|
spans.reduce((sum, r) => sum + fileLines.slice(r.start - 1, r.end).join('\n').length, 0);
|
||||||
|
/**
|
||||||
|
* Emit one file's section — header, the back-reference for whatever the
|
||||||
|
* agent already holds, and the fence for what is new. Every render path
|
||||||
|
* ends here so that the dedup bookkeeping (freed bytes, freed `maxFiles`
|
||||||
|
* slot, the session record, the diagnostic) is written in exactly one
|
||||||
|
* place and no path can forget a piece of it.
|
||||||
|
*
|
||||||
|
* A fully-held file emits its header and pointer and NO fence, and
|
||||||
|
* deliberately does not consume a `maxFiles` slot: that is half of where
|
||||||
|
* the reclaimed budget goes (the other half is `sourceSpent`, which the
|
||||||
|
* carry-forward pool hands down the rank order). Both send bytes to files
|
||||||
|
* the agent has NOT seen, which is the whole point.
|
||||||
|
*/
|
||||||
|
const emitFileSection = (opts: {
|
||||||
|
header: string;
|
||||||
|
/** Deduped source. Empty ⇒ the agent already holds all of it. */
|
||||||
|
body: string;
|
||||||
|
/** Spans `body` covers. */
|
||||||
|
ranges: ExploreLineRange[];
|
||||||
|
/** Spans replaced by the back-reference. */
|
||||||
|
covered: ExploreLineRange[];
|
||||||
|
/** Chars charged to `totalChars` on top of the body (fences, header). */
|
||||||
|
overhead: number;
|
||||||
|
mode: 'whole' | 'clusters' | 'focused' | 'skeleton';
|
||||||
|
clipped: boolean;
|
||||||
|
/** The undeduped render, kept for the no-new-source fallback. */
|
||||||
|
fullBody: string;
|
||||||
|
fullRanges: ExploreLineRange[];
|
||||||
|
}): void => {
|
||||||
|
// A remainder too small to be worth a fence is folded into the pointer
|
||||||
|
// (see MIN_DELTA_CHARS). Its ranges are then NOT recorded — the record
|
||||||
|
// must only ever claim source that was actually sent.
|
||||||
|
const folded = opts.covered.length > 0 && opts.body.length < EXPLORE_DEDUP.MIN_DELTA_CHARS;
|
||||||
|
const body = folded ? '' : opts.body;
|
||||||
|
const ranges = folded ? [] : opts.ranges;
|
||||||
|
const at = lines.length;
|
||||||
|
lines.push(opts.header, '');
|
||||||
|
if (opts.covered.length > 0) {
|
||||||
|
const pointer = formatBackReference(
|
||||||
|
filePath,
|
||||||
|
opts.covered,
|
||||||
|
symbolsInSpans(group.nodes, opts.covered),
|
||||||
|
{ partial: body.length > 0 },
|
||||||
|
);
|
||||||
|
lines.push(pointer, '');
|
||||||
|
totalChars += pointer.length + 2;
|
||||||
|
backReferencedFiles.push(filePath);
|
||||||
|
}
|
||||||
|
if (body.length > 0) {
|
||||||
|
lines.push('```' + lang, body, '```', '');
|
||||||
|
totalChars += body.length + opts.overhead;
|
||||||
|
sourceSpent += body.length;
|
||||||
|
newSourceChars += body.length;
|
||||||
|
diag?.recordRender(filePath, opts.mode, body.length, opts.clipped || opts.covered.length > 0);
|
||||||
|
if (opts.covered.length > 0) diag?.recordDedup(filePath, coveredChars(opts.covered), opts.covered);
|
||||||
|
noteEmitted(filePath, [...ranges, ...opts.covered], body.length, fingerprint);
|
||||||
|
renderedFilePaths.push(filePath);
|
||||||
|
filesIncluded++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Fully held. The section is the pointer; the slot and the bytes go to a
|
||||||
|
// file the agent has not seen. The spans are still recorded (at zero
|
||||||
|
// bytes) because the record means "source the agent HAS", not "bytes
|
||||||
|
// this call spent" — refreshing them keeps a long session from ageing
|
||||||
|
// them out of the retained window and re-serving them for nothing.
|
||||||
|
totalChars += opts.overhead;
|
||||||
|
diag?.recordRender(filePath, 'backref', 0, false);
|
||||||
|
diag?.recordDedup(filePath, coveredChars(opts.covered), opts.covered);
|
||||||
|
noteEmitted(filePath, opts.covered, 0, fingerprint);
|
||||||
|
renderedFilePaths.push(filePath);
|
||||||
|
if (!suppressedFallback && opts.fullBody.length > 0) {
|
||||||
|
suppressedFallback = {
|
||||||
|
filePath,
|
||||||
|
at,
|
||||||
|
replacing: lines.length - at,
|
||||||
|
section: [opts.header, '', '```' + lang, opts.fullBody, '```', ''],
|
||||||
|
sourceChars: opts.fullBody.length,
|
||||||
|
overhead: opts.overhead,
|
||||||
|
ranges: opts.fullRanges,
|
||||||
|
fingerprint,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// Disk-drift gate (#1474): every render branch below except whole-file
|
// Disk-drift gate (#1474): every render branch below except whole-file
|
||||||
// slices fileContent (CURRENT bytes) at INDEXED line ranges. Content is
|
// slices fileContent (CURRENT bytes) at INDEXED line ranges. Content is
|
||||||
@@ -4132,8 +4317,7 @@ export class ToolHandler {
|
|||||||
// Pass 2: render in line order — full body for chosen symbols, else the
|
// Pass 2: render in line order — full body for chosen symbols, else the
|
||||||
// signature line (capped, with a "+N more" tail so the structure map of a
|
// signature line (capped, with a "+N more" tail so the structure map of a
|
||||||
// god-file doesn't itself bloat the budget).
|
// god-file doesn't itself bloat the budget).
|
||||||
const skel: string[] = [];
|
const skel: Array<{ range: ExploreLineRange; text: string }> = [];
|
||||||
const skelRanges: ExploreLineRange[] = [];
|
|
||||||
let coveredUntil = 0; // skip symbols already inside an emitted body
|
let coveredUntil = 0; // skip symbols already inside an emitted body
|
||||||
let sigCount = 0, sigDropped = 0;
|
let sigCount = 0, sigDropped = 0;
|
||||||
const SIG_MAX = Math.max(12, budget.maxSymbolsInFileHeader * 2);
|
const SIG_MAX = Math.max(12, budget.maxSymbolsInFileHeader * 2);
|
||||||
@@ -4142,8 +4326,10 @@ export class ToolHandler {
|
|||||||
if (bodyIds.has(n.id)) {
|
if (bodyIds.has(n.id)) {
|
||||||
const end = n.endLine;
|
const end = n.endLine;
|
||||||
const body = fileLines.slice(n.startLine - 1, end).join('\n');
|
const body = fileLines.slice(n.startLine - 1, end).join('\n');
|
||||||
skel.push(exploreLineNumbersEnabled() ? numberSourceLines(body, n.startLine) : body);
|
skel.push({
|
||||||
skelRanges.push({ start: n.startLine, end });
|
range: { start: n.startLine, end },
|
||||||
|
text: withLineNumbers ? numberSourceLines(body, n.startLine) : body,
|
||||||
|
});
|
||||||
coveredUntil = end;
|
coveredUntil = end;
|
||||||
} else {
|
} else {
|
||||||
// Elide the body, emit the signature. node.startLine can point at a
|
// Elide the body, emit the signature. node.startLine can point at a
|
||||||
@@ -4156,13 +4342,15 @@ export class ToolHandler {
|
|||||||
if (sigCount >= SIG_MAX) { sigDropped++; continue; }
|
if (sigCount >= SIG_MAX) { sigDropped++; continue; }
|
||||||
const sig = (fileLines[lineNo - 1] || '').trim();
|
const sig = (fileLines[lineNo - 1] || '').trim();
|
||||||
if (sig) {
|
if (sig) {
|
||||||
skel.push(exploreLineNumbersEnabled() ? `${lineNo}\t${sig}` : sig);
|
skel.push({
|
||||||
skelRanges.push({ start: lineNo, end: lineNo });
|
range: { start: lineNo, end: lineNo },
|
||||||
|
text: withLineNumbers ? `${lineNo}\t${sig}` : sig,
|
||||||
|
});
|
||||||
sigCount++;
|
sigCount++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (sigDropped > 0) skel.push(`… +${sigDropped} more (signatures elided)`);
|
const sigTail = sigDropped > 0 ? `… +${sigDropped} more (signatures elided)` : '';
|
||||||
if (skel.length > 0) {
|
if (skel.length > 0) {
|
||||||
const names = [...new Set(group.nodes.filter(n => n.kind !== 'import' && n.kind !== 'export').map(n => n.name))]
|
const names = [...new Set(group.nodes.filter(n => n.kind !== 'import' && n.kind !== 'export').map(n => n.name))]
|
||||||
.slice(0, budget.maxSymbolsInFileHeader).join(', ');
|
.slice(0, budget.maxSymbolsInFileHeader).join(', ');
|
||||||
@@ -4175,14 +4363,25 @@ export class ToolHandler {
|
|||||||
const tag = bodyIds.size > 0
|
const tag = bodyIds.size > 0
|
||||||
? 'focused (the methods you named in full, the rest as signatures — codegraph_explore a signature by name for its body; do NOT Read)'
|
? 'focused (the methods you named in full, the rest as signatures — codegraph_explore a signature by name for its body; do NOT Read)'
|
||||||
: 'skeleton (signatures only — codegraph_explore a name for its full body; do NOT Read)';
|
: 'skeleton (signatures only — codegraph_explore a name for its full body; do NOT Read)';
|
||||||
lines.push(fileSectionHeader(filePath, `${names} · ${tag}`), '', '```' + lang, skel.join('\n'), '```', '');
|
// Dedup runs on the per-symbol parts, so a body the agent already has
|
||||||
totalChars += skel.join('\n').length + 120;
|
// becomes a pointer while the signature map around it survives intact
|
||||||
sourceSpent += skel.join('\n').length;
|
// (a one-line signature is far under MIN_COVERED_LINES and is never
|
||||||
// Always "clipped": the per-symbol view elides bodies by construction.
|
// withheld — the structure map is what makes this render legible).
|
||||||
diag?.recordRender(filePath, bodyIds.size > 0 ? 'focused' : 'skeleton', skel.join('\n').length, true);
|
const dd = dedupeSpans(skel);
|
||||||
noteEmitted(filePath, skelRanges, skel.join('\n').length);
|
const withTail = (parts: ReadonlyArray<{ text: string }>) =>
|
||||||
renderedFilePaths.push(filePath);
|
[...parts.map((p) => p.text), ...(sigTail ? [sigTail] : [])].join('\n');
|
||||||
filesIncluded++;
|
emitFileSection({
|
||||||
|
header: fileSectionHeader(filePath, `${names} · ${tag}`),
|
||||||
|
body: dd.parts.length > 0 ? withTail(dd.parts) : '',
|
||||||
|
ranges: dd.parts.map((p) => p.range),
|
||||||
|
covered: dd.covered,
|
||||||
|
overhead: 120,
|
||||||
|
mode: bodyIds.size > 0 ? 'focused' : 'skeleton',
|
||||||
|
// Always "clipped": the per-symbol view elides bodies by construction.
|
||||||
|
clipped: true,
|
||||||
|
fullBody: withTail(skel),
|
||||||
|
fullRanges: skel.map((p) => p.range),
|
||||||
|
});
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -4260,7 +4459,14 @@ export class ToolHandler {
|
|||||||
&& totalChars + fileContent.length + EXPLORE_ALLOCATION.FILE_OVERHEAD <= renderCeiling);
|
&& totalChars + fileContent.length + EXPLORE_ALLOCATION.FILE_OVERHEAD <= renderCeiling);
|
||||||
if (fileLines.length <= WHOLE_FILE_MAX_LINES && buysWhole) {
|
if (fileLines.length <= WHOLE_FILE_MAX_LINES && buysWhole) {
|
||||||
const body = fileContent.replace(/\n+$/, '');
|
const body = fileContent.replace(/\n+$/, '');
|
||||||
let wholeSection = exploreLineNumbersEnabled() ? numberSourceLines(body, 1) : body;
|
const wholeRange: ExploreLineRange = { start: 1, end: body.split('\n').length };
|
||||||
|
const fullSection = withLineNumbers ? numberSourceLines(body, 1) : body;
|
||||||
|
// The buy decision above was made on the file's FULL size on purpose: it
|
||||||
|
// asks "did this file's relevance earn all of itself", which dedup does
|
||||||
|
// not change. Dedup then only ever makes the render smaller, so a buy
|
||||||
|
// that was funded stays funded.
|
||||||
|
const ddWhole = dedupeSpans([{ range: wholeRange, text: fullSection }]);
|
||||||
|
const wholeSection = ddWhole.parts.map((p) => p.text).join(GAP_MARKER);
|
||||||
const uniqSymbols = [...new Set(
|
const uniqSymbols = [...new Set(
|
||||||
group.nodes
|
group.nodes
|
||||||
.filter(n => n.kind !== 'import' && n.kind !== 'export')
|
.filter(n => n.kind !== 'import' && n.kind !== 'export')
|
||||||
@@ -4281,14 +4487,18 @@ export class ToolHandler {
|
|||||||
diag?.recordSkip(filePath, 'budget-whole-file');
|
diag?.recordSkip(filePath, 'budget-whole-file');
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
lines.push(wholeHeader, '', '```' + lang, wholeSection, '```', '');
|
emitFileSection({
|
||||||
totalChars += wholeSection.length + 200;
|
header: wholeHeader,
|
||||||
sourceSpent += wholeSection.length;
|
body: wholeSection,
|
||||||
diag?.recordRender(filePath, 'whole', wholeSection.length, false);
|
// The whole file, minus any trailing blank lines the render trimmed.
|
||||||
// The whole file, minus any trailing blank lines the render trimmed.
|
ranges: ddWhole.parts.map((p) => p.range),
|
||||||
noteEmitted(filePath, [{ start: 1, end: body.split('\n').length }], wholeSection.length);
|
covered: ddWhole.covered,
|
||||||
renderedFilePaths.push(filePath);
|
overhead: 200,
|
||||||
filesIncluded++;
|
mode: 'whole',
|
||||||
|
clipped: false,
|
||||||
|
fullBody: fullSection,
|
||||||
|
fullRanges: [wholeRange],
|
||||||
|
});
|
||||||
if (fileStale) staleRendered.push(filePath);
|
if (fileStale) staleRendered.push(filePath);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -4441,10 +4651,6 @@ export class ToolHandler {
|
|||||||
// until the per-file char cap is hit. Truly enormous single clusters
|
// until the per-file char cap is hit. Truly enormous single clusters
|
||||||
// get tail-trimmed with a marker.
|
// get tail-trimmed with a marker.
|
||||||
const contextPadding = 3;
|
const contextPadding = 3;
|
||||||
const withLineNumbers = exploreLineNumbersEnabled();
|
|
||||||
// Language-neutral separator (no `//` — not a comment in Python, Ruby,
|
|
||||||
// etc.). With line numbers on, the line-number jump also signals the gap.
|
|
||||||
const GAP_MARKER = '\n\n... (gap) ...\n\n';
|
|
||||||
// An oversize spine method (the call path runs THROUGH a god-method — n8n's
|
// An oversize spine method (the call path runs THROUGH a god-method — n8n's
|
||||||
// processRunExecutionData is 962 lines) is windowed to its next-hop CALL site
|
// processRunExecutionData is 962 lines) is windowed to its next-hop CALL site
|
||||||
// plus the signature head, NOT dumped whole. Without this the cluster is too big
|
// plus the signature head, NOT dumped whole. Without this the cluster is too big
|
||||||
@@ -4453,42 +4659,50 @@ export class ToolHandler {
|
|||||||
// the spine's call still appears in context.
|
// the spine's call still appears in context.
|
||||||
const OVERSIZE_SPINE_LINES = 200;
|
const OVERSIZE_SPINE_LINES = 200;
|
||||||
const SPINE_WINDOW = 28; // lines each side of the next-hop call site
|
const SPINE_WINDOW = 28; // lines each side of the next-hop call site
|
||||||
// Returns the rendered text AND the line spans it covers. The spans are
|
// Returns the rendered text as SPAN-KEYED PARTS. Every part carries the
|
||||||
// what the session record is built from (CG-17): reporting them from the
|
// exact line range its text was sliced from, which two things depend on:
|
||||||
// same function that slices the source is what keeps the record honest —
|
// the session record (CG-17) — a record claiming lines it never sent would
|
||||||
// a second function mirroring these window/padding rules would drift, and
|
// withhold them from a later call, costing a Read — and cross-call dedup
|
||||||
// a record that claims lines it never sent withholds them from a later
|
// (CG-18), which rebuilds a part's text from a narrower span when the
|
||||||
// call, which costs a Read.
|
// agent already holds the rest. Both read the spans from the function that
|
||||||
|
// does the slicing; a second function mirroring these window/padding rules
|
||||||
|
// would drift.
|
||||||
|
type SectionPart = { range: ExploreLineRange; text: string };
|
||||||
|
const sectionText = (parts: ReadonlyArray<SectionPart>): string =>
|
||||||
|
parts.map((p) => p.text).join(GAP_MARKER);
|
||||||
const buildSection = (
|
const buildSection = (
|
||||||
c: { start: number; end: number; hasSpine?: boolean; spineCallLine?: number },
|
c: { start: number; end: number; hasSpine?: boolean; spineCallLine?: number },
|
||||||
): { text: string; ranges: ExploreLineRange[] } => {
|
): SectionPart[] => {
|
||||||
if (c.hasSpine && c.spineCallLine && (c.end - c.start + 1) > OVERSIZE_SPINE_LINES) {
|
if (c.hasSpine && c.spineCallLine && (c.end - c.start + 1) > OVERSIZE_SPINE_LINES) {
|
||||||
const call = c.spineCallLine;
|
const call = c.spineCallLine;
|
||||||
const winStart = Math.max(c.start, call - SPINE_WINDOW);
|
const winStart = Math.max(c.start, call - SPINE_WINDOW);
|
||||||
const winEnd = Math.min(c.end, call + SPINE_WINDOW);
|
const winEnd = Math.min(c.end, call + SPINE_WINDOW);
|
||||||
const parts: string[] = [];
|
const parts: SectionPart[] = [];
|
||||||
const spans: ExploreLineRange[] = [];
|
|
||||||
// Signature head, only when it sits clearly above the window (else the
|
// Signature head, only when it sits clearly above the window (else the
|
||||||
// window already covers the method opening).
|
// window already covers the method opening).
|
||||||
const headEnd = Math.min(c.start + 4, winStart - 2);
|
const headEnd = Math.min(c.start + 4, winStart - 2);
|
||||||
if (headEnd >= c.start) {
|
if (headEnd >= c.start) {
|
||||||
const head = fileLines.slice(c.start - 1, headEnd).join('\n');
|
const head = fileLines.slice(c.start - 1, headEnd).join('\n');
|
||||||
parts.push(withLineNumbers ? numberSourceLines(head, c.start) : head);
|
parts.push({
|
||||||
spans.push({ start: c.start, end: headEnd });
|
range: { start: c.start, end: headEnd },
|
||||||
|
text: withLineNumbers ? numberSourceLines(head, c.start) : head,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
const win = fileLines.slice(winStart - 1, winEnd).join('\n');
|
const win = fileLines.slice(winStart - 1, winEnd).join('\n');
|
||||||
parts.push(withLineNumbers ? numberSourceLines(win, winStart) : win);
|
parts.push({
|
||||||
spans.push({ start: winStart, end: winEnd });
|
range: { start: winStart, end: winEnd },
|
||||||
return { text: parts.join(GAP_MARKER), ranges: spans };
|
text: withLineNumbers ? numberSourceLines(win, winStart) : win,
|
||||||
|
});
|
||||||
|
return parts;
|
||||||
}
|
}
|
||||||
const startIdx = Math.max(0, c.start - 1 - contextPadding);
|
const startIdx = Math.max(0, c.start - 1 - contextPadding);
|
||||||
const endIdx = Math.min(fileLines.length, c.end + contextPadding);
|
const endIdx = Math.min(fileLines.length, c.end + contextPadding);
|
||||||
const slice = fileLines.slice(startIdx, endIdx).join('\n');
|
const slice = fileLines.slice(startIdx, endIdx).join('\n');
|
||||||
// startIdx is 0-based, so the slice's first line is line startIdx + 1.
|
// startIdx is 0-based, so the slice's first line is line startIdx + 1.
|
||||||
return {
|
return [{
|
||||||
|
range: { start: startIdx + 1, end: endIdx },
|
||||||
text: withLineNumbers ? numberSourceLines(slice, startIdx + 1) : slice,
|
text: withLineNumbers ? numberSourceLines(slice, startIdx + 1) : slice,
|
||||||
ranges: [{ start: startIdx + 1, end: endIdx }],
|
}];
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -4507,10 +4721,7 @@ export class ToolHandler {
|
|||||||
* body is never cut, and the members are chosen by the same importance the
|
* body is never cut, and the members are chosen by the same importance the
|
||||||
* cluster ranking uses. Returns null when nothing needed shrinking.
|
* cluster ranking uses. Returns null when nothing needed shrinking.
|
||||||
*/
|
*/
|
||||||
const shrinkCluster = (
|
const shrinkCluster = (c: ExploreCluster, cap: number): SectionPart[] | null => {
|
||||||
c: ExploreCluster,
|
|
||||||
cap: number,
|
|
||||||
): { text: string; ranges: ExploreLineRange[] } | null => {
|
|
||||||
if (c.members.length < 2) return null;
|
if (c.members.length < 2) return null;
|
||||||
const byImportance = [...c.members].sort((a, b) =>
|
const byImportance = [...c.members].sort((a, b) =>
|
||||||
b.importance - a.importance || (a.end - a.start) - (b.end - b.start) || a.start - b.start);
|
b.importance - a.importance || (a.end - a.start) - (b.end - b.start) || a.start - b.start);
|
||||||
@@ -4535,11 +4746,30 @@ export class ToolHandler {
|
|||||||
if (last && r.start <= last.end + gapThreshold) last.end = Math.max(last.end, r.end);
|
if (last && r.start <= last.end + gapThreshold) last.end = Math.max(last.end, r.end);
|
||||||
else merged.push({ start: r.start, end: r.end });
|
else merged.push({ start: r.start, end: r.end });
|
||||||
}
|
}
|
||||||
const sections = merged.map((m) => buildSection(m));
|
return merged.flatMap((m) => buildSection(m));
|
||||||
return {
|
};
|
||||||
text: sections.map((s) => s.text).join(GAP_MARKER),
|
|
||||||
ranges: sections.flatMap((s) => s.ranges),
|
/**
|
||||||
};
|
* One cluster's final parts: built, shrunk if it overruns `cap`, then
|
||||||
|
* passed through the session history (CG-18).
|
||||||
|
*
|
||||||
|
* The shrink decision reads the DEDUPED length on purpose. A cluster whose
|
||||||
|
* bytes the agent already holds costs this response nothing, so shrinking
|
||||||
|
* it on its raw size would drop new symbols to make room for source that
|
||||||
|
* is not being sent — spending the file's budget on nothing.
|
||||||
|
*/
|
||||||
|
const renderCluster = (
|
||||||
|
c: ExploreCluster,
|
||||||
|
cap: number,
|
||||||
|
): { parts: SectionPart[]; covered: ExploreLineRange[]; shrunk: boolean } => {
|
||||||
|
const base = dedupeSpans(buildSection(c));
|
||||||
|
if (sectionText(base.parts).length <= cap) {
|
||||||
|
return { parts: base.parts, covered: base.covered, shrunk: false };
|
||||||
|
}
|
||||||
|
const shrunk = shrinkCluster(c, cap);
|
||||||
|
if (shrunk === null) return { parts: base.parts, covered: base.covered, shrunk: false };
|
||||||
|
const dd = dedupeSpans(shrunk);
|
||||||
|
return { parts: dd.parts, covered: dd.covered, shrunk: true };
|
||||||
};
|
};
|
||||||
|
|
||||||
// Rank clusters for inclusion under the per-file cap. Entry-point
|
// Rank clusters for inclusion under the per-file cap. Entry-point
|
||||||
@@ -4586,23 +4816,28 @@ export class ToolHandler {
|
|||||||
// spine can't run away or starve co-flow files entirely.
|
// spine can't run away or starve co-flow files entirely.
|
||||||
const SPINE_CEILING = Math.min(Math.round(allowance * 1.5), headroom);
|
const SPINE_CEILING = Math.min(Math.round(allowance * 1.5), headroom);
|
||||||
const chosenIndices = new Set<number>();
|
const chosenIndices = new Set<number>();
|
||||||
// Shrunk renders for oversize clusters, by cluster index (CG-12). Computed
|
// Final renders (deduped, shrunk where oversize) by cluster index. Computed
|
||||||
// during selection and reused at emission so the two never disagree.
|
// during selection and reused at emission so the two never disagree.
|
||||||
const shrunkSections = new Map<number, { text: string; ranges: ExploreLineRange[] }>();
|
const renderedClusters = new Map<number, ReturnType<typeof renderCluster>>();
|
||||||
|
let anyClusterShrunk = false;
|
||||||
let projectedChars = 0;
|
let projectedChars = 0;
|
||||||
for (const rc of rankedClusters) {
|
for (const rc of rankedClusters) {
|
||||||
const sectionLen = buildSection(rc.c).text.length + (chosenIndices.size > 0 ? GAP_MARKER.length : 0);
|
|
||||||
// The top-ranked cluster is always taken — an empty file section sends the
|
// The top-ranked cluster is always taken — an empty file section sends the
|
||||||
// agent to Read, negating the savings. But "always taken" is not "taken at
|
// agent to Read, negating the savings. But "always taken" is not "taken at
|
||||||
// any size": when it overruns the reservation it is SHRUNK to the
|
// any size": when it overruns the reservation it is SHRUNK to the
|
||||||
// highest-importance whole symbol ranges inside it, so a single-cluster
|
// highest-importance whole symbol ranges inside it, so a single-cluster
|
||||||
// god-file spends its allotment instead of the whole response's.
|
// god-file spends its allotment instead of the whole response's. Later
|
||||||
if (chosenIndices.size === 0) {
|
// clusters are never shrunk — they either fit or wait for another call.
|
||||||
const cap = rc.c.hasSpine ? SPINE_CEILING : fileBudget;
|
const first = chosenIndices.size === 0;
|
||||||
const shrunk = sectionLen > cap ? shrinkCluster(rc.c, cap) : null;
|
const cap = rc.c.hasSpine ? SPINE_CEILING : fileBudget;
|
||||||
if (shrunk !== null) shrunkSections.set(rc.idx, shrunk);
|
const section = renderCluster(rc.c, first ? cap : Infinity);
|
||||||
|
const text = sectionText(section.parts);
|
||||||
|
const sectionLen = text.length + (!first && text.length > 0 ? GAP_MARKER.length : 0);
|
||||||
|
if (first) {
|
||||||
|
renderedClusters.set(rc.idx, section);
|
||||||
|
anyClusterShrunk = anyClusterShrunk || section.shrunk;
|
||||||
chosenIndices.add(rc.idx);
|
chosenIndices.add(rc.idx);
|
||||||
projectedChars += shrunk !== null ? shrunk.text.length : sectionLen;
|
projectedChars += sectionLen;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// A spine cluster (the rendered call path) is the flow answer — include it
|
// A spine cluster (the rendered call path) is the flow answer — include it
|
||||||
@@ -4611,6 +4846,7 @@ export class ToolHandler {
|
|||||||
const fits = projectedChars + sectionLen <= fileBudget;
|
const fits = projectedChars + sectionLen <= fileBudget;
|
||||||
const spineFits = rc.c.hasSpine && projectedChars + sectionLen <= SPINE_CEILING;
|
const spineFits = rc.c.hasSpine && projectedChars + sectionLen <= SPINE_CEILING;
|
||||||
if (!fits && !spineFits) continue;
|
if (!fits && !spineFits) continue;
|
||||||
|
renderedClusters.set(rc.idx, section);
|
||||||
chosenIndices.add(rc.idx);
|
chosenIndices.add(rc.idx);
|
||||||
projectedChars += sectionLen;
|
projectedChars += sectionLen;
|
||||||
}
|
}
|
||||||
@@ -4619,13 +4855,18 @@ export class ToolHandler {
|
|||||||
let fileSection = '';
|
let fileSection = '';
|
||||||
const allSymbols: string[] = [];
|
const allSymbols: string[] = [];
|
||||||
const sectionRanges: ExploreLineRange[] = [];
|
const sectionRanges: ExploreLineRange[] = [];
|
||||||
|
const coveredRanges: ExploreLineRange[] = [];
|
||||||
for (let i = 0; i < clusters.length; i++) {
|
for (let i = 0; i < clusters.length; i++) {
|
||||||
if (!chosenIndices.has(i)) continue;
|
if (!chosenIndices.has(i)) continue;
|
||||||
const cluster = clusters[i]!;
|
const cluster = clusters[i]!;
|
||||||
const section = shrunkSections.get(i) ?? buildSection(cluster);
|
const section = renderedClusters.get(i)!;
|
||||||
if (fileSection.length > 0) fileSection += GAP_MARKER;
|
const text = sectionText(section.parts);
|
||||||
fileSection += section.text;
|
if (text.length > 0) {
|
||||||
sectionRanges.push(...section.ranges);
|
if (fileSection.length > 0) fileSection += GAP_MARKER;
|
||||||
|
fileSection += text;
|
||||||
|
}
|
||||||
|
sectionRanges.push(...section.parts.map((p) => p.range));
|
||||||
|
coveredRanges.push(...section.covered);
|
||||||
allSymbols.push(...cluster.symbols);
|
allSymbols.push(...cluster.symbols);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4635,7 +4876,7 @@ export class ToolHandler {
|
|||||||
// method is useless (the agent just Reads the rest for the other half), which
|
// method is useless (the agent just Reads the rest for the other half), which
|
||||||
// is the very fallback explore exists to prevent. A pathological file is
|
// is the very fallback explore exists to prevent. A pathological file is
|
||||||
// bounded by the cluster SELECTION above + the total hard ceiling.
|
// bounded by the cluster SELECTION above + the total hard ceiling.
|
||||||
if (chosenIndices.size < clusters.length || shrunkSections.size > 0) {
|
if (chosenIndices.size < clusters.length || anyClusterShrunk) {
|
||||||
anyFileTrimmed = true;
|
anyFileTrimmed = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4673,19 +4914,62 @@ export class ToolHandler {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
lines.push(fileHeader);
|
// The undeduped render of the same clusters, needed only if this file ends
|
||||||
lines.push('');
|
// up fully back-referenced AND the whole call finds nothing new to say —
|
||||||
lines.push('```' + lang);
|
// see `suppressedFallback`. Built lazily: on every other call it is dead
|
||||||
lines.push(fileSection);
|
// weight.
|
||||||
lines.push('```');
|
const fullClusterParts = fileSection.length === 0
|
||||||
lines.push('');
|
? clusters.flatMap((c, i) => (chosenIndices.has(i) ? buildSection(c) : []))
|
||||||
|
: [];
|
||||||
|
emitFileSection({
|
||||||
|
header: fileHeader,
|
||||||
|
body: fileSection,
|
||||||
|
ranges: sectionRanges,
|
||||||
|
covered: mergeRanges(coveredRanges),
|
||||||
|
overhead: 200,
|
||||||
|
mode: 'clusters',
|
||||||
|
clipped: chosenIndices.size < clusters.length,
|
||||||
|
fullBody: sectionText(fullClusterParts),
|
||||||
|
fullRanges: fullClusterParts.map((p) => p.range),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
totalChars += fileSection.length + 200;
|
// Anti-abandonment restore (CG-18). Dedup withheld everything and nothing new
|
||||||
sourceSpent += fileSection.length;
|
// took its place — the response would be pointers only, which is the shape
|
||||||
diag?.recordRender(filePath, 'clusters', fileSection.length, chosenIndices.size < clusters.length);
|
// that reads as "codegraph found nothing" and sends the agent to Read for
|
||||||
noteEmitted(filePath, sectionRanges, fileSection.length);
|
// good. Put the top suppressed file back, in full, and keep its pointer off.
|
||||||
renderedFilePaths.push(filePath);
|
// Deliberately checked against `newSourceChars` (source THIS call emitted)
|
||||||
filesIncluded++;
|
// rather than the response length: the flow and blast-radius sections are
|
||||||
|
// always there, and they are not what makes a response feel sufficient.
|
||||||
|
// Cast, not annotation: the only writer is the render loop's `emitFileSection`
|
||||||
|
// closure, which TypeScript's flow analysis cannot see, so it narrows the
|
||||||
|
// variable to `null` here and the truthiness check below would be `never`.
|
||||||
|
const restore = suppressedFallback as SuppressedFallback | null;
|
||||||
|
if (newSourceChars === 0 && restore) {
|
||||||
|
if (totalChars + restore.sourceChars + restore.overhead <= renderCeiling) {
|
||||||
|
lines.splice(restore.at, restore.replacing, ...restore.section);
|
||||||
|
totalChars += restore.sourceChars + restore.overhead;
|
||||||
|
sourceSpent += restore.sourceChars;
|
||||||
|
newSourceChars += restore.sourceChars;
|
||||||
|
filesIncluded++;
|
||||||
|
const idx = backReferencedFiles.indexOf(restore.filePath);
|
||||||
|
if (idx >= 0) backReferencedFiles.splice(idx, 1);
|
||||||
|
emittedByFile.set(restore.filePath, {
|
||||||
|
ranges: [...restore.ranges],
|
||||||
|
bytes: restore.sourceChars,
|
||||||
|
fingerprint: restore.fingerprint,
|
||||||
|
});
|
||||||
|
diag?.recordRender(restore.filePath, 'clusters', restore.sourceChars, false);
|
||||||
|
diag?.recordDedup(restore.filePath, 0, []);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The back-reference convention, stated once where the verbatim guarantee is
|
||||||
|
// (#1474 does the same for drift). Without it a pointer reads as an
|
||||||
|
// apology for missing source rather than as an index into source the agent
|
||||||
|
// already has.
|
||||||
|
if (backReferencedFiles.length > 0) {
|
||||||
|
lines[verbatimHeaderIdx] += ` (Files marked **"Already sent earlier in this conversation"** are not repeated: their source came back on an earlier codegraph_explore call in THIS conversation and the file has not changed since, so that copy is exact and current — scroll back for it rather than re-fetching or Reading.)`;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Drift epilogue (#1474). The "verbatim / do not Read" guarantee above
|
// Drift epilogue (#1474). The "verbatim / do not Read" guarantee above
|
||||||
@@ -4833,12 +5117,22 @@ export class ToolHandler {
|
|||||||
// Session record (CG-17): only the files that SURVIVED the hard ceiling —
|
// Session record (CG-17): only the files that SURVIVED the hard ceiling —
|
||||||
// a section the truncation dropped was never delivered, and recording it
|
// a section the truncation dropped was never delivered, and recording it
|
||||||
// would let a later call withhold source the agent has never seen.
|
// would let a later call withhold source the agent has never seen.
|
||||||
|
//
|
||||||
|
// A back-referenced file records its spans at ZERO bytes (CG-18) — it is
|
||||||
|
// still source the agent holds for this file, which is what the record
|
||||||
|
// means; dropping it would let the span age out of the retained window and
|
||||||
|
// be re-served for nothing.
|
||||||
const emittedFiles: ExploreFileEmission[] = [];
|
const emittedFiles: ExploreFileEmission[] = [];
|
||||||
let sourceBytes = 0;
|
let sourceBytes = 0;
|
||||||
for (const fp of survivors) {
|
for (const fp of survivors) {
|
||||||
const emitted = emittedByFile.get(fp);
|
const emitted = emittedByFile.get(fp);
|
||||||
if (!emitted || emitted.bytes <= 0) continue;
|
if (!emitted || emitted.ranges.length === 0) continue;
|
||||||
emittedFiles.push({ path: fp, ranges: emitted.ranges, bytes: emitted.bytes });
|
emittedFiles.push({
|
||||||
|
path: fp,
|
||||||
|
ranges: emitted.ranges,
|
||||||
|
bytes: emitted.bytes,
|
||||||
|
fingerprint: emitted.fingerprint,
|
||||||
|
});
|
||||||
sourceBytes += emitted.bytes;
|
sourceBytes += emitted.bytes;
|
||||||
}
|
}
|
||||||
return this.exploreResult(finalText, {
|
return this.exploreResult(finalText, {
|
||||||
|
|||||||
Reference in New Issue
Block a user