From b37f191f5a6b2346adf364462b33ad094f7136d9 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Mon, 3 Aug 2026 22:51:18 -0500 Subject: [PATCH] feat(mcp): per-file allocation diagnostic for explore (CG-4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit How codegraph_explore divides its byte envelope among files was unobservable — you could read a response and guess, but not say "this file took 16% and that one took 20%." Nothing else in the budget- allocation epic is measurable without that. CODEGRAPH_EXPLORE_DEBUG now emits one report per explore call (stderr table, stderr JSON, or a JSONL sidecar path). Per file: relevance score, graph mass, term hits, ranking flags, render mode, bytes allocated vs delivered, both shares, and whether it was clipped — plus why a ranked candidate never rendered. Totals cover envelope vs maxOutputChars vs the hard ceiling, the source/meta split, the selection funnel, and the score floor and relevance-gate thresholds applied. Allocated and delivered are reported separately on purpose: they diverge exactly when the 25K ceiling truncates, and conflating them is how a dropped trailing file goes unnoticed. Off by default and byte-identical when off — it ships in the product binary, and a diagnostic that perturbs the response by one byte would invalidate every A/B taken with it on. ExploreDiagnostics.start() returns null unless the env var is set, so every call site is a `diag?.` no-op. Baseline recorded in docs/design/explore-budget-allocation.md: on this repo, src/mcp/tools.ts gets 15.8% of the envelope while three weakly- relevant agent-eval scripts take 61% between them — despite tools.ts carrying 5.4x the score and 2.6x the graph mass of any of them. Small files ship whole; the large answer file is clipped at maxCharsPerFile. Rank ordering is correct and buys nothing. The loop also allocated 23,193 chars against an 18,000 budget. Co-Authored-By: Claude Opus 5 --- __tests__/explore-diagnostics.test.ts | 294 +++++++++++++ docs/design/explore-budget-allocation.md | 102 +++++ src/mcp/explore-diagnostics.ts | 528 +++++++++++++++++++++++ src/mcp/tools.ts | 72 +++- 4 files changed, 990 insertions(+), 6 deletions(-) create mode 100644 __tests__/explore-diagnostics.test.ts create mode 100644 docs/design/explore-budget-allocation.md create mode 100644 src/mcp/explore-diagnostics.ts diff --git a/__tests__/explore-diagnostics.test.ts b/__tests__/explore-diagnostics.test.ts new file mode 100644 index 0000000..e8b925c --- /dev/null +++ b/__tests__/explore-diagnostics.test.ts @@ -0,0 +1,294 @@ +/** + * Per-file allocation diagnostic for codegraph_explore (CG-4). + * + * The instrument ships in the product binary, so the load-bearing property is + * NOT what it reports — it's that it reports NOTHING unless asked. An explore + * response is the agent's context; a diagnostic that perturbs it by one byte + * invalidates every A/B measurement taken with it on, which is the exact thing + * the rest of the budget-allocation work depends on. + * + * So the first block pins byte-identical output across on/off, and only then + * do we assert the report's shape and internal consistency. + */ +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { ToolHandler } from '../src/mcp/tools'; +import { attributeSourceBytes } from '../src/mcp/explore-diagnostics'; +import CodeGraph from '../src/index'; + +const DEBUG_ENV = 'CODEGRAPH_EXPLORE_DEBUG'; + +/** Restore the env var to "unset" — `delete` matters; '' is a distinct case. */ +function clearDebugEnv(): void { + delete process.env[DEBUG_ENV]; +} + +describe('attributeSourceBytes', () => { + it('attributes a fenced block to the file section header above it', () => { + const text = [ + '**Exploration: x**', + '', + '**`src/a.ts`** — foo(function)', + '', + '```typescript', + '1\tconst a = 1;', + '2\tconst b = 2;', + '```', + '', + '**`src/b.ts`** — bar(function)', + '', + '```typescript', + '1\tconst c = 3;', + '```', + '', + ].join('\n'); + const bytes = attributeSourceBytes(text); + expect(bytes.get('src/a.ts')).toBe('1\tconst a = 1;\n2\tconst b = 2;'.length); + expect(bytes.get('src/b.ts')).toBe('1\tconst c = 3;'.length); + }); + + it('sums multiple fenced blocks under one file header', () => { + const text = [ + '**`src/a.ts`** — foo(function)', + '', + '```ts', + 'aa', + '```', + '', + '```ts', + 'bbb', + '```', + ].join('\n'); + expect(attributeSourceBytes(text).get('src/a.ts')).toBe('aa'.length + 'bbb'.length); + }); + + it('counts an unterminated block — the ceiling can cut mid-fence', () => { + const text = ['**`src/a.ts`** — foo(function)', '', '```ts', 'x'.repeat(40)].join('\n'); + expect(attributeSourceBytes(text).get('src/a.ts')).toBe(40); + }); + + it('returns nothing for text with no file sections', () => { + expect(attributeSourceBytes('No relevant code found for "zzz"').size).toBe(0); + expect(attributeSourceBytes('').size).toBe(0); + }); +}); + +describe('codegraph_explore allocation diagnostic', () => { + let testDir: string; + let sidecarDir: string; + let cg: CodeGraph; + let handler: ToolHandler; + + const QUERY = 'Session method helper callSession'; + + beforeAll(async () => { + testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-explore-diag-')); + sidecarDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-explore-diag-out-')); + const srcDir = path.join(testDir, 'src'); + fs.mkdirSync(srcDir); + + // One fat file plus several small callers, so the render loop exercises + // more than one allocation branch (clusters for the fat file, whole-file + // for the small ones) and there is a real per-file split to report. + const fatLines: string[] = ['export class Session {']; + for (let i = 0; i < 30; i++) { + fatLines.push(` method${i}(arg: string): string {`); + fatLines.push(` return this.helper${i}(arg) + "${i}";`); + fatLines.push(` }`); + fatLines.push(` private helper${i}(arg: string): string {`); + fatLines.push(` return arg.repeat(${i + 1});`); + fatLines.push(` }`); + } + fatLines.push('}'); + fs.writeFileSync(path.join(srcDir, 'session.ts'), fatLines.join('\n')); + + for (let i = 0; i < 6; i++) { + fs.writeFileSync( + path.join(srcDir, `support${i}.ts`), + `import { Session } from './session';\n` + + `export function callSession${i}(s: Session) {\n` + + ` return s.method${i}('hi');\n` + + `}\n`, + ); + } + + clearDebugEnv(); + cg = CodeGraph.initSync(testDir, { config: { include: ['**/*.ts'], exclude: [] } }); + await cg.indexAll(); + handler = new ToolHandler(cg); + }); + + afterEach(() => { + clearDebugEnv(); + vi.restoreAllMocks(); + }); + + afterAll(() => { + clearDebugEnv(); + if (cg) cg.destroy(); + for (const dir of [testDir, sidecarDir]) { + if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + const explore = async (): Promise => { + const result = await handler.execute('codegraph_explore', { query: QUERY }); + return result.content?.[0]?.text ?? ''; + }; + + it('produces byte-identical output whether the diagnostic is on or off', async () => { + clearDebugEnv(); + const off = await explore(); + expect(off.length).toBeGreaterThan(0); + + // Sanity: the tool itself is deterministic, so a difference below is + // attributable to the diagnostic and not to explore's own variance. + expect(await explore()).toBe(off); + + vi.spyOn(process.stderr, 'write').mockImplementation((() => true) as typeof process.stderr.write); + const sidecar = path.join(sidecarDir, 'identical.jsonl'); + for (const value of ['1', 'json', sidecar]) { + process.env[DEBUG_ENV] = value; + const on = await explore(); + clearDebugEnv(); + expect(on).toBe(off); + } + }); + + it('writes nothing to stderr when the env var is unset', async () => { + clearDebugEnv(); + const writes: string[] = []; + vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { + writes.push(String(chunk)); + return true; + }) as typeof process.stderr.write); + await explore(); + expect(writes.join('')).toBe(''); + }); + + it('stays off for every falsy env value', async () => { + const writes: string[] = []; + vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { + writes.push(String(chunk)); + return true; + }) as typeof process.stderr.write); + for (const value of ['', '0', 'false', 'off', 'no', 'OFF', ' 0 ']) { + process.env[DEBUG_ENV] = value; + await explore(); + } + expect(writes.join('')).toBe(''); + }); + + it('prints a per-file table to stderr when enabled', async () => { + const writes: string[] = []; + vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { + writes.push(String(chunk)); + return true; + }) as typeof process.stderr.write); + + process.env[DEBUG_ENV] = '1'; + await explore(); + const out = writes.join(''); + + expect(out).toContain('codegraph explore diagnostic'); + // Totals: envelope vs budget, and the file-selection funnel with its floor. + expect(out).toMatch(/envelope [\d,]+ chars delivered · [\d,]+ allocated of [\d,]+ budget/); + expect(out).toMatch(/hard ceiling [\d,]+/); + expect(out).toMatch(/files [\d,]+ grouped .*past score floor \(>=\d+\).*in output \(maxFiles \d+\)/); + // Per-file columns. + expect(out).toMatch(/#\s+alloc%\s+deliv%\s+bytes\s+score\s+graph\s+hits\s+flags\s+render\s+file/); + expect(out).toContain('src/session.ts'); + expect(out).toMatch(/\d+\.\d%/); + }); + + it('appends one JSON report per call to a sidecar path', async () => { + const sidecar = path.join(sidecarDir, 'reports.jsonl'); + process.env[DEBUG_ENV] = sidecar; + await explore(); + await explore(); + clearDebugEnv(); + + const rows = fs.readFileSync(sidecar, 'utf-8').trim().split('\n'); + expect(rows).toHaveLength(2); + + const report = JSON.parse(rows[0]!); + expect(report.tool).toBe('codegraph_explore'); + expect(report.query).toBe(QUERY); + + // Totals the task asks for: envelope vs maxOutputChars, files considered + // vs included, and the score floor that was applied. + expect(report.budget.maxOutputChars).toBeGreaterThan(0); + expect(report.envelope.chars).toBeGreaterThan(0); + expect(report.selection.scoreFloor).toBeGreaterThan(0); + expect(report.selection.filesGrouped).toBeGreaterThanOrEqual(report.selection.filesPastScoreFloor); + expect(report.selection.filesPastScoreFloor).toBeGreaterThanOrEqual(report.selection.filesRanked); + expect(report.selection.filesRanked).toBeGreaterThanOrEqual(report.selection.filesInFinalOutput); + expect(report.selection.filesInFinalOutput).toBeGreaterThan(0); + expect(report.selection.filesInFinalOutput).toBeLessThanOrEqual(report.budget.maxFiles); + + // Per-file: score, bytes, share, clipped, spine. + const shown = report.files.filter((f: { finalChars: number }) => f.finalChars > 0); + expect(shown.length).toBeGreaterThan(0); + for (const f of shown) { + expect(typeof f.path).toBe('string'); + expect(typeof f.score).toBe('number'); + expect(typeof f.graphScore).toBe('number'); + expect(typeof f.clipped).toBe('boolean'); + expect(typeof f.spine).toBe('boolean'); + expect(f.finalChars).toBeGreaterThan(0); + expect(f.share).toBeGreaterThan(0); + expect(f.share).toBeLessThanOrEqual(1); + expect(f.render).toBeTruthy(); + } + expect(shown.some((f: { path: string }) => f.path === 'src/session.ts')).toBe(true); + }); + + it('attributes the envelope consistently — per-file bytes sum to the reported source total', async () => { + const sidecar = path.join(sidecarDir, 'consistency.jsonl'); + process.env[DEBUG_ENV] = sidecar; + const text = await explore(); + clearDebugEnv(); + + const report = JSON.parse(fs.readFileSync(sidecar, 'utf-8').trim()); + expect(report.envelope.chars).toBe(text.length); + + const summed = report.files.reduce( + (s: number, f: { finalChars: number }) => s + f.finalChars, 0, + ); + expect(summed).toBe(report.envelope.sourceChars); + expect(report.envelope.sourceChars + report.envelope.metaChars).toBe(report.envelope.chars); + // Shares are fractions of the delivered envelope, so they can't exceed it. + const shareSum = report.files.reduce((s: number, f: { share: number }) => s + f.share, 0); + expect(shareSum).toBeLessThanOrEqual(1.0001); + expect(shareSum).toBeCloseTo(report.envelope.sourceShare, 3); + }); + + it('survives an unwritable sink without failing the explore call', async () => { + clearDebugEnv(); + const expected = await explore(); + + // A directory is never a valid append target. + process.env[DEBUG_ENV] = sidecarDir; + const result = await handler.execute('codegraph_explore', { query: QUERY }); + clearDebugEnv(); + + expect(result.isError).toBeFalsy(); + expect(result.content?.[0]?.text).toBe(expected); + }); + + it('records a report even when explore finds nothing', async () => { + const sidecar = path.join(sidecarDir, 'empty.jsonl'); + process.env[DEBUG_ENV] = sidecar; + const result = await handler.execute('codegraph_explore', { + query: 'zzzznonexistentsymbolzzzz', + }); + clearDebugEnv(); + + expect(result.content?.[0]?.text).toContain('No relevant code found'); + const report = JSON.parse(fs.readFileSync(sidecar, 'utf-8').trim()); + expect(report.note).toContain('no relevant code found'); + expect(report.files).toEqual([]); + }); +}); diff --git a/docs/design/explore-budget-allocation.md b/docs/design/explore-budget-allocation.md new file mode 100644 index 0000000..4371362 --- /dev/null +++ b/docs/design/explore-budget-allocation.md @@ -0,0 +1,102 @@ +# Explore budget allocation — the instrument and the baseline + +`codegraph_explore` has a fixed byte envelope (`getExploreOutputBudget().maxOutputChars`, +hard-capped at 25K so the host never externalizes the result). **How that envelope gets +divided among files** is decided by a long chain of gates, tiers and caps spread across +`handleExplore` — and until CG-4 that chain was unobservable. You could read an explore +response and guess; you could not say "this file took 16% and that one took 20%." + +This document covers the diagnostic that makes it measurable, and the baseline it recorded. + +## The diagnostic + +Set `CODEGRAPH_EXPLORE_DEBUG` and every `codegraph_explore` call (MCP tool or +`codegraph explore` CLI) emits one report: + +| value | sink | +|---|---| +| `1` / `true` / `on` / `yes` / `stderr` | human-readable table on stderr | +| `json` | one pretty-printed JSON report on stderr | +| anything else | treated as a path — one JSON report per line, appended (JSONL) | +| unset / `0` / `false` / `off` / `no` / empty | **off** | + +Per file it reports: relevance score, graph (RWR) mass, distinct query-term hits, ranking +flags (named / entry / central / spine / low-value / generated), render mode, bytes of +source allocated, bytes actually delivered, both shares, and whether it was clipped. For +files that never rendered it reports why (`max-files`, `budget-90pct`, `budget-whole-file`, +`budget-clusters`, `unreadable`, `no-ranges`). Totals cover the envelope (delivered vs +allocated vs `maxOutputChars` vs hard ceiling), the source/meta split, the file-selection +funnel at each stage, and the thresholds applied (score floor, graph-relevance gate). + +**It is off by default and produces byte-identical output when off** — it ships in the +product binary, and a diagnostic that perturbs the response by one byte would invalidate +every A/B measurement taken with it on. `ExploreDiagnostics.start()` returns `null` unless +the env var is set, so every call site is a `diag?.` no-op. Pinned by +`__tests__/explore-diagnostics.test.ts`. + +Two envelope numbers, deliberately kept separate: + +- **allocated** — what the render loop chose to emit, before the final hard-ceiling cut. + This is the allocator's own decision, and the number budget work is about. +- **delivered** — what the agent actually received. + +They diverge exactly when the ceiling truncates. Conflating them is how a dropped trailing +file goes unnoticed. + +## Baseline (2026-08-03, this repo at `main`) + +``` +codegraph explore "how does explore allocate its output budget across files" --path . +``` + +469 files indexed → small tier (`maxOutputChars` 18,000, `maxCharsPerFile` 3,800, +`defaultMaxFiles` 5). Envelope: **23,196 delivered / 23,193 allocated against an 18,000 +budget — 29% over**, absorbed only because the 25K hard ceiling sits above it. + +| # | share | bytes | score | graph | hits | flags | render | file | +|---|---|---|---|---|---|---|---|---| +| 4 | 21.2% | 4,928 | 10 | 0.125 | 1 | entry | whole | `scripts/agent-eval/offload-eval-hook.mjs` | +| 5 | 20.1% | 4,665 | 10 | 0.125 | 1 | entry | whole | `scripts/agent-eval/offload-eval-metrics.mjs` | +| 3 | 19.8% | 4,585 | 22 | 0.125 | 1 | entry central | whole | `scripts/agent-eval/parse-session.mjs` | +| 1 | **15.8%** | 3,659 | **54** | **0.322** | **4** | entry central | clusters* | `src/mcp/tools.ts` | +| 2 | 15.0% | 3,479 | 34 | 0.082 | 2 | entry | clusters* | `src/index.ts` | + +\* clipped. Ranked but never rendered: `scripts/agent-eval/offload-eval-cost.mjs` (#6) and +`src/resolution/lru-cache.ts` (#7), both cut by `maxFiles`. + +Files: 17 grouped → 10 past the score floor (≥3) → 10 past the low-value filter → 7 past +the relevance gate (graph ≥ 0.0193, 6% of max 0.3215) → 5 in the output. + +### What the baseline shows + +**Score does not drive allocation.** `src/mcp/tools.ts` — the file that actually answers +the query — carries 5.4× the relevance score, 2.6× the graph mass and 4× the term hits of +any `.mjs` script, and gets a *smaller* share than each of them. The three agent-eval +scripts take **61%** of the envelope between them; the answer file takes 16%. + +The mechanism is that the two allocation paths are decided by **file size, not relevance**: +a small file clears `WHOLE_FILE_MAX_LINES`/`WHOLE_FILE_MAX_CHARS` and ships entirely, while +a large file falls through to cluster selection and is clipped at `maxCharsPerFile`. So a +weakly-relevant 130-line script gets 100% of itself; the strongly-relevant 5,000-line file +gets 3,800 chars. Rank ordering is correct (tools.ts sorts #1) and buys nothing, because +rank has no effect on how many bytes a file receives. + +**The envelope is over-subscribed.** 23,193 allocated against an 18,000 budget means the +per-file caps do not compose into the total cap; the total is enforced only by the 25K +ceiling silently dropping whole trailing sections. Under a slightly different index state +(one more candidate file) the same query allocated 27,518 chars and the ceiling dropped a +7,678-char section — the single largest allocation in the response — with the only trace +being the truncation notice at the end. + +This is the gap the rest of the epic closes: relevance-proportional allocation with a +relative cliff (CG-12), on top of scoring that stops rewarding incidental name collisions +(CG-10). + +### Reproducing + +The query explores this repo, so **uncommitted edits to `src/mcp/tools.ts` change the +result** — the index picks them up and scores shift (the same query on the CG-4 working +tree reported tools.ts at 13–19% depending on the sync state). Measure against a clean +tree: restore `src/mcp/tools.ts` from `main`, remove `src/mcp/explore-diagnostics.ts`, +`codegraph sync`, then run the built `dist/` binary (which still carries the instrument). +Restore afterwards. diff --git a/src/mcp/explore-diagnostics.ts b/src/mcp/explore-diagnostics.ts new file mode 100644 index 0000000..210a5cb --- /dev/null +++ b/src/mcp/explore-diagnostics.ts @@ -0,0 +1,528 @@ +/** + * Per-file allocation diagnostic for `codegraph_explore` (CG-4). + * + * The explore response is a fixed byte envelope (`budget.maxOutputChars`, hard- + * capped at 25K so the host never externalizes the result). WHICH files fill it, + * and in what proportion, is decided by a long chain of gates, tiers and caps + * spread across `handleExplore`. That chain is currently unobservable: you can + * read the output and guess, but you cannot say "this file took 16% of the + * envelope and that one took 21%" without hand-counting. + * + * This module is the instrument. Enabled by `CODEGRAPH_EXPLORE_DEBUG`, it + * records, for one explore call: + * - per candidate file: relevance score, graph (RWR) mass, distinct query-term + * hits, ranking flags, render mode, bytes of source actually emitted, that + * file's share of the final envelope, whether it was clipped, whether it + * carries a flow-spine symbol — and for the ones that didn't render, why; + * - totals: envelope vs `maxOutputChars` vs the hard ceiling, source bytes vs + * meta-text overhead, files considered at each filter stage, and the score + * floor / relevance-gate thresholds that were applied. + * + * HARD CONSTRAINT — this ships in the product binary: when the env var is unset + * the diagnostic must not exist. `start()` returns `null`, every call site is a + * `diag?.` no-op, and the agent-facing response is byte-identical. The + * diagnostic never mutates render state, and every method is wrapped so a bug in + * here can never fail an explore call. + * + * Sinks (value of `CODEGRAPH_EXPLORE_DEBUG`): + * `1` / `true` / `on` / `yes` / `stderr` → human-readable table on stderr + * `json` → one JSON object on stderr + * anything else → treated as a path; one JSON object + * per line appended (JSONL sidecar) + */ + +import { appendFileSync } from 'fs'; + +/** How a file's source was rendered into the response. */ +export type ExploreRenderMode = + | 'whole' // whole-file window + | 'clusters' // ranked contiguous clusters + | 'focused' // per-symbol view, named/spine bodies full + | 'skeleton' // per-symbol view, signatures only + | 'stale-omitted' // drifted on disk; source deliberately withheld + | 'dropped'; // rendered into `lines` but cut by the final hard ceiling + +/** Why a ranked candidate never reached the output. */ +export type ExploreSkipReason = + | 'max-files' // maxFiles reached before this file + | 'budget-90pct' // incidental file past the 90%-of-budget soft stop + | 'budget-whole-file' // incidental whole-file render wouldn't fit + | 'budget-clusters' // incidental cluster render wouldn't fit + | 'unreadable' // outside root, missing, or read error + | 'no-ranges'; // no renderable line ranges in this file + +/** Ranking inputs for one candidate file, captured before the render loop. */ +export interface ExploreCandidateMeta { + rank: number; + score: number; + graphScore: number; + termHits: number; + nodes: number; + named: boolean; + central: boolean; + entry: boolean; + spine: boolean; + lowValue: boolean; + generated: boolean; +} + +interface FileRecord extends ExploreCandidateMeta { + path: string; + render?: ExploreRenderMode; + /** Source chars the render loop handed to `lines` (pre-final-truncation). */ + emittedChars: number; + /** Source chars present in the FINAL text — authoritative, truncation-aware. */ + finalChars: number; + /** Share of the DELIVERED envelope, as a fraction (0–1). */ + share: number; + /** Share of what the render loop ALLOCATED, before the hard-ceiling cut. */ + allocatedShare: number; + clipped: boolean; + skipped?: ExploreSkipReason; +} + +interface StageCounts { + /** Files with at least one gathered node. */ + grouped: number; + /** Survived the `group.score >= scoreFloor` filter. */ + pastScoreFloor: number; + /** Survived the test/spec/icon/i18n hard-exclude. */ + pastLowValueFilter: number; + /** Survived the graph-relevance gate. */ + pastRelevanceGate: number; +} + +/** Budget fields the diagnostic reports. Structural, to avoid a cyclic import. */ +interface BudgetShape { + maxOutputChars: number; + maxCharsPerFile: number; + defaultMaxFiles: number; +} + +/** One file's line in the report. Also the JSONL sidecar's per-file shape. */ +export interface ExploreDiagnosticFile extends ExploreCandidateMeta { + path: string; + render: ExploreRenderMode | null; + skipped: ExploreSkipReason | null; + clipped: boolean; + emittedChars: number; + finalChars: number; + share: number; + allocatedShare: number; +} + +/** The full report — one per explore call, JSON-serialized to the sink. */ +export interface ExploreDiagnosticReport { + tool: 'codegraph_explore'; + query: string; + projectRoot: string; + indexedFileCount: number; + note?: string; + budget: { + maxOutputChars: number; + maxCharsPerFile: number; + maxFiles: number; + hardCeiling: number; + }; + envelope: { + /** Chars actually returned to the agent (post-truncation). */ + chars: number; + /** Chars the render loop produced, BEFORE the hard-ceiling cut. */ + allocatedChars: number; + overBudget: boolean; + truncated: boolean; + sourceChars: number; + sourceShare: number; + metaChars: number; + metaShare: number; + }; + selection: { + scoreFloor: number; + maxGraph: number; + graphGateThreshold: number; + graphGateApplied: boolean; + filesGrouped: number; + filesPastScoreFloor: number; + filesPastLowValueFilter: number; + filesRanked: number; + filesRenderedByLoop: number; + filesInFinalOutput: number; + }; + files: ExploreDiagnosticFile[]; +} + +type Sink = + | { kind: 'stderr'; json: boolean } + | { kind: 'file'; path: string }; + +const OFF = new Set(['', '0', 'false', 'off', 'no']); +const STDERR_TABLE = new Set(['1', 'true', 'on', 'yes', 'stderr']); + +/** + * Resolve the sink from the environment. `null` means the diagnostic is off — + * read per call (not memoized) so a test can toggle it between invocations. + */ +function resolveSink(): Sink | null { + const raw = process.env.CODEGRAPH_EXPLORE_DEBUG; + if (raw === undefined) return null; + const value = raw.trim(); + const lower = value.toLowerCase(); + if (OFF.has(lower)) return null; + if (STDERR_TABLE.has(lower)) return { kind: 'stderr', json: false }; + if (lower === 'json') return { kind: 'stderr', json: true }; + return { kind: 'file', path: value }; +} + +const num = (n: number) => Math.round(n).toLocaleString('en-US'); +const pct = (f: number) => `${(f * 100).toFixed(1)}%`; + +export class ExploreDiagnostics { + private readonly files = new Map(); + private readonly stages: StageCounts = { + grouped: 0, pastScoreFloor: 0, pastLowValueFilter: 0, pastRelevanceGate: 0, + }; + private scoreFloor = 0; + private maxGraph = 0; + private graphGateThreshold = 0; + private graphGateApplied = false; + private note = ''; + + private constructor( + private readonly sink: Sink, + private readonly query: string, + private readonly projectRoot: string, + private readonly budget: BudgetShape, + private readonly maxFiles: number, + private readonly indexedFileCount: number, + ) {} + + /** + * Returns `null` when `CODEGRAPH_EXPLORE_DEBUG` is unset/off — the whole + * instrument then costs one env read per explore call and nothing else. + */ + static start( + query: string, + projectRoot: string, + budget: BudgetShape, + maxFiles: number, + indexedFileCount: number, + ): ExploreDiagnostics | null { + try { + const sink = resolveSink(); + if (!sink) return null; + return new ExploreDiagnostics(sink, query, projectRoot, budget, maxFiles, indexedFileCount); + } catch { + return null; + } + } + + /** Candidate count after the initial `group.score >= floor` filter. */ + setScoreFloor(floor: number, grouped: number, kept: number): void { + this.scoreFloor = floor; + this.stages.grouped = grouped; + this.stages.pastScoreFloor = kept; + this.stages.pastLowValueFilter = kept; + this.stages.pastRelevanceGate = kept; + } + + /** Candidate count after the test/spec/icon/i18n hard-exclude. */ + setLowValueFiltered(kept: number): void { + this.stages.pastLowValueFilter = kept; + this.stages.pastRelevanceGate = kept; + } + + /** Graph-relevance gate: threshold, whether it actually pruned, what survived. */ + setRelevanceGate(maxGraph: number, threshold: number, applied: boolean, kept: number): void { + this.maxGraph = maxGraph; + this.graphGateThreshold = threshold; + this.graphGateApplied = applied; + this.stages.pastRelevanceGate = kept; + } + + /** Record one ranked candidate's scoring inputs, in final sort order. */ + noteCandidate(path: string, meta: ExploreCandidateMeta): void { + this.files.set(path, { + path, ...meta, + emittedChars: 0, finalChars: 0, share: 0, allocatedShare: 0, clipped: false, + }); + } + + /** A candidate rendered source into the response. */ + recordRender(path: string, render: ExploreRenderMode, sourceChars: number, clipped: boolean): void { + const rec = this.files.get(path); + if (!rec) return; + rec.render = render; + rec.emittedChars = sourceChars; + rec.clipped = clipped; + rec.skipped = undefined; + } + + /** + * A candidate was passed over before rendering. First reason wins — the + * blanket `max-files` sweep must not overwrite a file's specific reason. + */ + recordSkip(path: string, reason: ExploreSkipReason): void { + const rec = this.files.get(path); + if (!rec || rec.render || rec.skipped) return; + rec.skipped = reason; + } + + /** Explore returned early (no subgraph). Emits a minimal record. */ + finishEmpty(reason: string): void { + this.note = reason; + this.emit(this.buildReport('', 0, 0, 0)); + } + + /** + * Final pass: attribute the FINAL text's bytes back to files (so the hard + * ceiling's truncation is reflected in what each file actually delivered), + * then emit. + * + * `allocatedChars` is the pre-truncation length — the size the render loop + * *chose*. Reporting both is the point: the allocator's decision and the + * agent's delivered payload diverge exactly when the ceiling cuts, and + * conflating them is how a dropped trailing file goes unnoticed. + */ + finish(finalText: string, allocatedChars: number, hardCeiling: number, filesIncluded: number): void { + try { + const perFile = attributeSourceBytes(finalText); + const envelope = finalText.length; + for (const rec of this.files.values()) { + rec.finalChars = perFile.get(rec.path) ?? 0; + rec.share = envelope > 0 ? rec.finalChars / envelope : 0; + rec.allocatedShare = allocatedChars > 0 ? rec.emittedChars / allocatedChars : 0; + // Rendered into `lines` but absent from the final text → the hard + // ceiling dropped its whole section. + if (rec.render && rec.render !== 'stale-omitted' && rec.finalChars === 0) { + rec.render = 'dropped'; + rec.clipped = true; + } + } + this.emit(this.buildReport(finalText, allocatedChars, hardCeiling, filesIncluded)); + } catch { + // A diagnostic must never fail an explore call. + } + } + + private buildReport( + finalText: string, + allocatedChars: number, + hardCeiling: number, + filesIncluded: number, + ): ExploreDiagnosticReport { + const envelope = finalText.length; + const records = [...this.files.values()]; + const rendered = records.filter((r) => r.finalChars > 0); + const sourceChars = rendered.reduce((s, r) => s + r.finalChars, 0); + return { + tool: 'codegraph_explore', + query: this.query, + projectRoot: this.projectRoot, + indexedFileCount: this.indexedFileCount, + note: this.note || undefined, + budget: { + maxOutputChars: this.budget.maxOutputChars, + maxCharsPerFile: this.budget.maxCharsPerFile, + maxFiles: this.maxFiles, + hardCeiling, + }, + envelope: { + chars: envelope, + allocatedChars, + overBudget: allocatedChars > this.budget.maxOutputChars, + truncated: allocatedChars > hardCeiling, + sourceChars, + sourceShare: envelope > 0 ? sourceChars / envelope : 0, + metaChars: envelope - sourceChars, + metaShare: envelope > 0 ? (envelope - sourceChars) / envelope : 0, + }, + selection: { + scoreFloor: this.scoreFloor, + maxGraph: this.maxGraph, + graphGateThreshold: this.graphGateThreshold, + graphGateApplied: this.graphGateApplied, + filesGrouped: this.stages.grouped, + filesPastScoreFloor: this.stages.pastScoreFloor, + filesPastLowValueFilter: this.stages.pastLowValueFilter, + filesRanked: this.stages.pastRelevanceGate, + filesRenderedByLoop: filesIncluded, + filesInFinalOutput: rendered.length, + }, + files: records + .slice() + .sort((a, b) => b.emittedChars - a.emittedChars || b.finalChars - a.finalChars || a.rank - b.rank) + .map((r) => ({ + path: r.path, + rank: r.rank, + score: r.score, + graphScore: round6(r.graphScore), + termHits: r.termHits, + nodes: r.nodes, + named: r.named, + central: r.central, + entry: r.entry, + spine: r.spine, + lowValue: r.lowValue, + generated: r.generated, + render: r.render ?? null, + skipped: r.skipped ?? null, + clipped: r.clipped, + emittedChars: r.emittedChars, + finalChars: r.finalChars, + share: round6(r.share), + allocatedShare: round6(r.allocatedShare), + })), + }; + } + + private emit(report: ExploreDiagnosticReport): void { + try { + if (this.sink.kind === 'file') { + appendFileSync(this.sink.path, JSON.stringify(report) + '\n', 'utf-8'); + return; + } + if (this.sink.json) { + process.stderr.write(JSON.stringify(report, null, 2) + '\n'); + return; + } + process.stderr.write(renderTable(report) + '\n'); + } catch { + // Unwritable sidecar / closed stderr must not fail the explore call. + } + } +} + +const round6 = (n: number) => Math.round(n * 1e6) / 1e6; + +/** + * Attribute the final response's source bytes back to files by walking the + * rendered markdown: a ``**`path`**`` section header followed by a fenced code + * block. Reading the FINAL text (rather than trusting the render loop's running + * total) is what makes the numbers truthful — it accounts for the hard-ceiling + * truncation that can drop whole trailing sections after they were "emitted". + * + * Line numbering is on by default, so a source line that is itself a ``` fence + * arrives as `42\t```` and cannot close the block early. + */ +export function attributeSourceBytes(finalText: string): Map { + const out = new Map(); + if (!finalText) return out; + const lines = finalText.split('\n'); + let current: string | null = null; + let inFence = false; + let acc: string[] = []; + const flush = () => { + if (current && acc.length > 0) { + out.set(current, (out.get(current) ?? 0) + acc.join('\n').length); + } + acc = []; + }; + for (const line of lines) { + if (!inFence) { + const header = /^\*\*`([^`]+)`\*\*/.exec(line); + if (header) { + current = header[1]!; + continue; + } + if (current && line.startsWith('```')) { + inFence = true; + continue; + } + continue; + } + if (line === '```') { + inFence = false; + flush(); + continue; + } + acc.push(line); + } + // Unterminated fence (final-ceiling truncation cut mid-block): count it. + if (inFence) flush(); + return out; +} + +/** Human-readable stderr rendering of the JSON report. */ +export function renderTable(report: ExploreDiagnosticReport): string { + const { budget, envelope: env, selection: sel, files } = report; + + const out: string[] = []; + out.push(''); + out.push(`codegraph explore diagnostic — "${report.query}"`); + out.push(` project ${report.projectRoot} · ${num(report.indexedFileCount)} files indexed`); + if (report.note) out.push(` note: ${report.note}`); + out.push( + ` envelope ${num(env.chars)} chars delivered · ${num(env.allocatedChars)} allocated` + + ` of ${num(budget.maxOutputChars)} budget (hard ceiling ${num(budget.hardCeiling)})` + + `${env.overBudget ? ' [over budget]' : ''}${env.truncated ? ' [TRUNCATED]' : ''}`, + ); + out.push( + ` source ${num(env.sourceChars)} (${pct(env.sourceShare)})` + + ` · meta ${num(env.metaChars)} (${pct(env.metaShare)})` + + ` · per-file cap ${num(budget.maxCharsPerFile)}`, + ); + out.push( + ` files ${num(sel.filesGrouped)} grouped` + + ` → ${num(sel.filesPastScoreFloor)} past score floor (>=${sel.scoreFloor})` + + ` → ${num(sel.filesPastLowValueFilter)} past low-value filter` + + ` → ${num(sel.filesRanked)} past relevance gate` + + ` → ${num(sel.filesInFinalOutput)} in output (maxFiles ${num(budget.maxFiles)})`, + ); + out.push( + ` relevance gate ${sel.graphGateApplied ? 'applied' : 'not applied'}` + + ` at graph >= ${sel.graphGateThreshold.toFixed(5)} (6% of max ${sel.maxGraph.toFixed(5)})`, + ); + out.push(''); + + // Allocated (not delivered) is the allocator's own decision — the number the + // budget work is about. Delivered is what the agent got. They differ only + // when the ceiling truncated; showing both makes that divergence obvious. + const shown = files.filter((f) => f.emittedChars > 0 || f.finalChars > 0); + if (shown.length > 0) { + out.push(' # alloc% deliv% bytes score graph hits flags render file'); + for (const f of shown) { + out.push( + ' ' + + String(f.rank).padStart(2) + ' ' + + pct(f.allocatedShare).padStart(6) + ' ' + + pct(f.share).padStart(6) + ' ' + + num(f.emittedChars).padStart(7) + ' ' + + String(f.score).padStart(5) + ' ' + + f.graphScore.toFixed(5).padStart(7) + ' ' + + String(f.termHits).padStart(4) + ' ' + + flagString(f).padEnd(19) + ' ' + + ((f.render ?? '-') + (f.clipped ? '*' : '')).padEnd(9) + ' ' + + f.path, + ); + } + out.push(' (bytes = source allocated by the render loop; deliv% = 0 means the hard ceiling dropped the section)'); + out.push(' (* = clipped: some source in this file was elided, windowed, or its section dropped)'); + } else { + out.push(' (no file source in the final output)'); + } + + const skipped = files.filter((f) => f.emittedChars === 0 && f.finalChars === 0); + if (skipped.length > 0) { + out.push(''); + out.push(' ranked but never rendered:'); + for (const f of skipped.slice(0, 15)) { + out.push( + ` #${String(f.rank).padStart(2)} ${f.path} — ${f.skipped ?? f.render ?? 'not reached'}` + + ` (score ${f.score}, graph ${f.graphScore.toFixed(5)}, hits ${f.termHits})`, + ); + } + if (skipped.length > 15) out.push(` … and ${skipped.length - 15} more`); + } + return out.join('\n'); +} + +function flagString(f: ExploreDiagnosticFile): string { + const flags: string[] = []; + if (f.named) flags.push('named'); + if (f.entry) flags.push('entry'); + if (f.central) flags.push('central'); + if (f.spine) flags.push('spine'); + if (f.lowValue) flags.push('low-value'); + if (f.generated) flags.push('generated'); + return flags.join(' ') || '-'; +} diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 08308ac..d1fcbc9 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -42,6 +42,7 @@ import { clamp, validatePathWithinRoot, validateProjectPath, isConfigLeafNode, C import { isGeneratedFile } from '../extraction/generated-detection'; import { scanDynamicDispatch } from './dynamic-boundaries'; import { getUpdateNotice } from '../upgrade/update-check'; +import { ExploreDiagnostics } from './explore-diagnostics'; /** * An expected, recoverable "codegraph can't serve this" condition — most @@ -2630,13 +2631,20 @@ export class ToolHandler { // largest-tier defaults if stats aren't available, which preserves // pre-#185 behavior for callers that hit the rare stats failure. let budget: ExploreOutputBudget; + let indexedFileCount = -1; try { - budget = getExploreOutputBudget(cg.getStats().fileCount); + indexedFileCount = cg.getStats().fileCount; + budget = getExploreOutputBudget(indexedFileCount); } catch { budget = getExploreOutputBudget(Infinity); } const maxFiles = clamp((args.maxFiles as number) || budget.defaultMaxFiles, 1, 20); + // Per-file allocation diagnostic (CG-4). `null` unless CODEGRAPH_EXPLORE_DEBUG + // is set — every `diag?.` below is then a no-op and the response is + // byte-identical. It only OBSERVES: it must never feed back into rendering. + const diag = ExploreDiagnostics.start(query, projectRoot, budget, maxFiles, indexedFileCount); + // Step 1: Find relevant context with generous parameters. // Use a large maxNodes budget — explore has its own 35k char output limit // that prevents context bloat, so more nodes just means better coverage @@ -2649,6 +2657,7 @@ export class ToolHandler { }); if (subgraph.nodes.size === 0) { + diag?.finishEmpty('no relevant code found — empty subgraph'); return this.textResult(`No relevant code found for "${query}"`); } @@ -2913,7 +2922,9 @@ export class ToolHandler { } // Only include files that have entry points or nodes directly connected to entry points - let relevantFiles = [...fileGroups.entries()].filter(([, group]) => group.score >= 3); + const SCORE_FLOOR = 3; + let relevantFiles = [...fileGroups.entries()].filter(([, group]) => group.score >= SCORE_FLOOR); + diag?.setScoreFloor(SCORE_FLOOR, fileGroups.size, relevantFiles.length); // Extract query terms for relevance checking const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length >= 3); @@ -2955,6 +2966,7 @@ export class ToolHandler { relevantFiles = nonLow; } } + diag?.setLowValueFiltered(relevantFiles.length); } // Secondary signal: how many DISTINCT query terms each file matches (path + @@ -3057,6 +3069,9 @@ export class ToolHandler { || (fileTermHits.get(fp) ?? 0) >= 2, ); if (gated.length >= 2) relevantFiles = gated; + diag?.setRelevanceGate(maxGraph, maxGraph * 0.06, gated.length >= 2, relevantFiles.length); + } else { + diag?.setRelevanceGate(maxGraph, 0, false, relevantFiles.length); } // Sort files: graph-central first, then distinct-term match, then the @@ -3202,6 +3217,27 @@ export class ToolHandler { // off-spine peers skeletonize. const flow = this.buildFlowFromNamedSymbols(cg, query); + // Snapshot every ranked candidate's scoring inputs, in final sort order, so + // the diagnostic can show what each file's share of the envelope was BOUGHT + // with (score, graph mass, term hits, flags) — not just what it cost. + if (diag) { + sortedFiles.forEach(([fp, group], i) => { + diag.noteCandidate(fp, { + rank: i + 1, + score: group.score, + graphScore: fileGraphScore.get(fp) ?? 0, + termHits: fileTermHits.get(fp) ?? 0, + nodes: group.nodes.length, + named: namedSeedFiles.has(fp), + central: centralFiles.has(fp), + entry: entryFiles.has(fp), + spine: group.nodes.some((n) => flow.pathNodeIds.has(n.id)), + lowValue: isLowValue(fp), + generated: isGeneratedFile(fp), + }); + }); + } + // Polymorphic-sibling detector for adaptive sizing. A class that implements/ // extends a supertype shared by >= MIN_SIBLINGS classes is one of many // INTERCHANGEABLE implementations (OkHttp's 14 `: Interceptor` classes — @@ -3278,7 +3314,10 @@ export class ToolHandler { const staleOmitted: string[] = []; for (const [filePath, group] of sortedFiles) { - if (filesIncluded >= maxFiles) break; + if (filesIncluded >= maxFiles) { + if (diag) for (const [fp] of sortedFiles) diag.recordSkip(fp, 'max-files'); + break; + } // A file DEFINES a named/spine symbol (the answer) vs merely references the // flow. Past 90% budget, stop pulling INCIDENTAL files — but keep scanning // for necessary ones, which render even past the cap (bounded by maxFiles). @@ -3287,15 +3326,22 @@ export class ToolHandler { // validate-logic file (Alamofire's Validation.swift). const fileNecessary = group.nodes.some(n => entryNodeIds.has(n.id) || flow.pathNodeIds.has(n.id) || flow.uniqueNamedNodeIds.has(n.id)); - if (!fileNecessary && totalChars > budget.maxOutputChars * 0.9) continue; + if (!fileNecessary && totalChars > budget.maxOutputChars * 0.9) { + diag?.recordSkip(filePath, 'budget-90pct'); + continue; + } const absPath = validatePathWithinRoot(projectRoot, filePath); - if (!absPath || !existsSync(absPath)) continue; + if (!absPath || !existsSync(absPath)) { + diag?.recordSkip(filePath, 'unreadable'); + continue; + } let fileContent: string; try { fileContent = readFileSync(absPath, 'utf-8'); } catch { + diag?.recordSkip(filePath, 'unreadable'); continue; } @@ -3419,6 +3465,8 @@ export class ToolHandler { : 'skeleton (signatures only — codegraph_explore a name for its full body; do NOT Read)'; lines.push(fileSectionHeader(filePath, `${names} · ${tag}`), '', '```' + lang, skel.join('\n'), '```', ''); totalChars += skel.join('\n').length + 120; + // Always "clipped": the per-symbol view elides bodies by construction. + diag?.recordRender(filePath, bodyIds.size > 0 ? 'focused' : 'skeleton', skel.join('\n').length, true); renderedFilePaths.push(filePath); filesIncluded++; continue; @@ -3470,10 +3518,12 @@ export class ToolHandler { // fit is skipped; a necessary one (below) renders in full. Half a file // forces the Read this is meant to prevent. anyFileTrimmed = true; + diag?.recordSkip(filePath, 'budget-whole-file'); continue; } lines.push(wholeHeader, '', '```' + lang, wholeSection, '```', ''); totalChars += wholeSection.length + 200; + diag?.recordRender(filePath, 'whole', wholeSection.length, false); renderedFilePaths.push(filePath); filesIncluded++; if (fileStale) staleRendered.push(filePath); @@ -3492,6 +3542,7 @@ export class ToolHandler { '', ); totalChars += 260; + diag?.recordRender(filePath, 'stale-omitted', 0, true); continue; } @@ -3567,7 +3618,10 @@ export class ToolHandler { ranges.sort((a, b) => a.start - b.start); - if (ranges.length === 0) continue; + if (ranges.length === 0) { + diag?.recordSkip(filePath, 'no-ranges'); + continue; + } const gapThreshold = budget.gapThreshold; const clusters: Array<{ start: number; end: number; symbols: string[]; score: number; maxImportance: number; hasSpine: boolean; spineCallLine?: number }> = []; @@ -3764,6 +3818,7 @@ export class ToolHandler { // Keep scanning for necessary files (which bypass this cap and render in // full, bounded by the hard ceiling). anyFileTrimmed = true; + diag?.recordSkip(filePath, 'budget-clusters'); continue; } @@ -3775,6 +3830,7 @@ export class ToolHandler { lines.push(''); totalChars += fileSection.length + 200; + diag?.recordRender(filePath, 'clusters', fileSection.length, chosenIndices.size < clusters.length); renderedFilePaths.push(filePath); filesIncluded++; } @@ -3896,6 +3952,10 @@ export class ToolHandler { : `Found ${subgraph.nodes.size} symbol${subgraph.nodes.size === 1 ? '' : 's'} across ${fileGroups.size} file${fileGroups.size === 1 ? '' : 's'}.`; finalText = finalText.replace(SUMMARY_SENTINEL, summaryLine); + // Emit the allocation diagnostic from the FINAL text, so per-file bytes and + // shares account for the hard-ceiling truncation above (CG-4). + diag?.finish(finalText, output.length, hardCeiling, filesIncluded); + return this.textResult(finalText); }