diff --git a/__tests__/explore-path-pinning.test.ts b/__tests__/explore-path-pinning.test.ts new file mode 100644 index 0000000..8ec15e1 --- /dev/null +++ b/__tests__/explore-path-pinning.test.ts @@ -0,0 +1,105 @@ +/** + * End-to-end gate for query-path pinning + the segment-vocab supplement + + * variable seeding, on the bug that motivated all three: an agent named a + * SvelteKit route file by exact path plus behavior words ("scrollToBottom, + * onscroll, atBottom tracking") and got back neither the file's scroll code + * nor the file itself at full weight — the bracketed path was tokenizer + * shrapnel (`runId` seeded as a named symbol, every sibling `+page` admitted) + * and the camelCase scroll symbols were FTS-opaque. + * + * The fixture mirrors that shape in plain TS (bracket/paren directories are + * the crux, not the language): a target file under + * `src/routes/m/projects/[id]/runs/[runId]/` holding `feedAtBottom` / + * `handleFeedScroll` / `pinFeedIfNearBottom`, a decoy chat-window page under + * a `(protected)` route group, and a runs-store decoy defining `runId` and + * `Scope` — the two symbols that headlined the original junk blast radius. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; + +const FIXTURE = 'explore-path-pinning'; +const TARGET = 'src/routes/m/projects/[id]/runs/[runId]/+page.ts'; +const DECOY_CHAT = 'src/routes/(protected)/chat-window/+page.ts'; + +let dir: string; +let cg: CodeGraph; + +async function explore(query: string): Promise { + const res = await new ToolHandler(cg).execute('codegraph_explore', { query }); + return res.content?.[0]?.text ?? ''; +} + +/** The response renders a source section for `file`. */ +const hasSection = (response: string, file: string): boolean => + response.includes('**`' + file + '`'); + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-path-pin-')); + fs.cpSync(path.join(__dirname, 'fixtures', FIXTURE), dir, { recursive: true }); + fs.rmSync(path.join(dir, '.codegraph'), { recursive: true, force: true }); + cg = CodeGraph.initSync(dir); + await cg.indexAll(); +}, 180_000); + +afterAll(() => { + cg?.destroy(); + if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('fixture shape — if this rots, the gates below mean nothing', () => { + it('indexes the bracketed-path target with its scroll symbols', () => { + const names = cg.getNodesInFile(TARGET).map((n) => n.name); + expect(names).toContain('feedAtBottom'); + expect(names).toContain('handleFeedScroll'); + expect(names).toContain('pinFeedIfNearBottom'); + }); +}); + +describe('path pinning (fix 1)', () => { + it('a pure-path query renders the named file and says it was pinned', async () => { + const out = await explore(TARGET); + expect(hasSection(out, TARGET)).toBe(true); + expect(out).toContain('pinned from the query'); + }); + + it('the original bug-shaped query renders the pinned file, not path shrapnel', async () => { + const out = await explore( + `run page auto-scroll to bottom logic in ${TARGET} — scrollToBottom, onscroll, atBottom tracking`, + ); + expect(hasSection(out, TARGET)).toBe(true); + // The path fragments must not seed: `runId` (runs-store decoy) and the + // bracketed segment's namesakes headlined the original junk blast radius. + const blast = out.split('**Relationships**')[0]!; + expect(blast).not.toMatch(/`runId` \(src\/lib\/runs-store\.ts/); + // The chat decoy MAY render — it genuinely holds scroll-pinning code the + // segment supplement now finds — but the pinned file must rank first. + // (Pre-fix, `+page`/`runs` shrapnel admitted the siblings ABOVE the named + // file and the envelope truncated it.) + const decoyAt = out.indexOf('**`' + DECOY_CHAT + '`'); + const targetAt = out.indexOf('**`' + TARGET + '`'); + expect(targetAt).toBeGreaterThan(-1); + if (decoyAt !== -1) expect(targetAt).toBeLessThan(decoyAt); + }); + + it('an unresolvable path is reported, not silently dropped', async () => { + const out = await explore('crash in src/routes/gone/missing-page.ts on load'); + expect(out).toContain('No indexed file uniquely matches'); + expect(out).toContain('src/routes/gone/missing-page.ts'); + }); +}); + +describe('segment supplement + variable seeding (fixes 2–3)', () => { + it('word-level scroll terms reach the camelCase scroll code without a path', async () => { + const out = await explore('feed auto-scroll to bottom pinning behavior'); + expect(hasSection(out, TARGET)).toBe(true); + }); + + it('a camel infix naming only $state-style variables still finds their file', async () => { + const out = await explore('where does the atBottom flag get reset'); + expect(hasSection(out, TARGET)).toBe(true); + }); +}); diff --git a/__tests__/explore-pinned-allocation.test.ts b/__tests__/explore-pinned-allocation.test.ts new file mode 100644 index 0000000..e093c8a --- /dev/null +++ b/__tests__/explore-pinned-allocation.test.ts @@ -0,0 +1,83 @@ +/** + * Pinned files in `allocateExploreBudget` (see query-paths.ts): a file the + * query named by PATH must survive every allocation guard. Its score is + * whatever the path-stripped query happened to match — for a pure-path query, + * nearly nothing — so without the pinned floor the proportional split would + * fund the one file the agent explicitly asked for worst of all, and the + * cliff would zero it outright. + */ +import { describe, it, expect } from 'vitest'; +import { allocateExploreBudget, getExploreOutputBudget, EXPLORE_ALLOCATION } from '../src/mcp/tools'; +import type { ExploreAllocationCandidate } from '../src/mcp/tools'; + +const cand = ( + path: string, + score: number, + extra: Partial = {}, +): ExploreAllocationCandidate => ({ path, score, worth: 1, spine: false, ...extra }); + +const budget = getExploreOutputBudget(1000); + +describe('allocateExploreBudget — pinned files', () => { + it('never cliffs a pinned file, however low it scores', () => { + const { allowances, cliffed } = allocateExploreBudget( + [ + cand('pinned.svelte', 0.1, { pinned: true }), + cand('hub.ts', 200), + cand('noise.ts', 0.1), + ], + budget, + 8, + ); + expect(cliffed).toContain('noise.ts'); + expect(cliffed).not.toContain('pinned.svelte'); + expect(allowances.has('pinned.svelte')).toBe(true); + }); + + it('funds a pinned file at least as well as the strongest candidate', () => { + const { allowances } = allocateExploreBudget( + [ + cand('pinned.svelte', 0.5, { pinned: true }), + cand('hub.ts', 300), + cand('helper.ts', 40), + ], + budget, + 8, + ); + expect(allowances.get('pinned.svelte')!).toBeGreaterThanOrEqual(allowances.get('hub.ts')!); + expect(allowances.get('pinned.svelte')!).toBeGreaterThan(allowances.get('helper.ts')!); + }); + + it('keeps pinned files through the affordability trim', () => { + // Smallest tier: affordable = floor(13000 / (MIN_CHARS + FILE_OVERHEAD)) = 14 + // slots. 18 equal-weight candidates admitted → the trim must cut 4. The + // pinned file sits last with a TIED weight (the pinned floor lifts it to + // the top weight), so the stable by-weight sort would slice it off — only + // the explicit spine/pinned keep saves it. + const tiny = getExploreOutputBudget(10); + const fleet = Array.from({ length: 17 }, (_, i) => cand(`f${i}.ts`, 100)); + fleet.push(cand('pinned.svelte', 0.1, { pinned: true })); + const { allowances, cliffed } = allocateExploreBudget(fleet, tiny, 18); + expect(allowances.has('pinned.svelte')).toBe(true); + expect(cliffed).not.toContain('pinned.svelte'); + expect(allowances.size).toBeLessThan(18); + }); + + it('an all-pinned zero-score call still allocates (pure-path query)', () => { + const { allowances, pool } = allocateExploreBudget( + [cand('a.svelte', 0, { pinned: true }), cand('b.svelte', 0, { pinned: true })], + budget, + 8, + ); + expect(pool).toBeGreaterThan(0); + expect(allowances.get('a.svelte')!).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.MIN_CHARS); + expect(allowances.get('b.svelte')!).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.MIN_CHARS); + }); + + it('unpinned behavior is unchanged when no candidate is pinned', () => { + const before = allocateExploreBudget( + [cand('a.ts', 40), cand('b.ts', 10)], budget, 8, + ); + expect(before.allowances.get('a.ts')!).toBeGreaterThan(before.allowances.get('b.ts')!); + }); +}); diff --git a/__tests__/fixtures/explore-path-pinning/package.json b/__tests__/fixtures/explore-path-pinning/package.json new file mode 100644 index 0000000..c17caa9 --- /dev/null +++ b/__tests__/fixtures/explore-path-pinning/package.json @@ -0,0 +1,5 @@ +{ + "name": "explore-path-pinning-fixture", + "version": "1.0.0", + "private": true +} diff --git a/__tests__/fixtures/explore-path-pinning/src/lib/runs-store.ts b/__tests__/fixtures/explore-path-pinning/src/lib/runs-store.ts new file mode 100644 index 0000000..b3a39d8 --- /dev/null +++ b/__tests__/fixtures/explore-path-pinning/src/lib/runs-store.ts @@ -0,0 +1,31 @@ +/** In-memory registry of task runs, keyed by run id. */ + +export interface Scope { + projectId: string; + label: string; +} + +export const runId = 'run-000'; + +const runs = new Map(); + +export function registerRun(id: string, scope: Scope): void { + runs.set(id, { id, scope, status: 'queued' }); +} + +export function getRun(id: string): { id: string; scope: Scope; status: string } | null { + return runs.get(id) ?? null; +} + +export function listRuns(scope: Scope): string[] { + return [...runs.values()] + .filter((r) => r.scope.projectId === scope.projectId) + .map((r) => r.id); +} + +export function stopRun(id: string): boolean { + const run = runs.get(id); + if (!run) return false; + run.status = 'cancelled'; + return true; +} diff --git a/__tests__/fixtures/explore-path-pinning/src/routes/(protected)/chat-window/+page.ts b/__tests__/fixtures/explore-path-pinning/src/routes/(protected)/chat-window/+page.ts new file mode 100644 index 0000000..3023ba4 --- /dev/null +++ b/__tests__/fixtures/explore-path-pinning/src/routes/(protected)/chat-window/+page.ts @@ -0,0 +1,28 @@ +/** Detached chat window page — session presence + streaming state. */ + +let chatAtBottom = true; +let isStreaming = false; +let messages: string[] = []; + +export function handleMessagesScroll(distance: number): void { + chatAtBottom = distance < 50; +} + +export function sendMessage(text: string): void { + messages = [...messages, text]; + isStreaming = true; +} + +export function stopResponse(): void { + isStreaming = false; +} + +export function redock(): void { + messages = []; + isStreaming = false; + chatAtBottom = true; +} + +export function chatSnapshot(): { messages: string[]; streaming: boolean } { + return { messages: [...messages], streaming: isStreaming }; +} diff --git a/__tests__/fixtures/explore-path-pinning/src/routes/m/projects/[id]/runs/[runId]/+page.ts b/__tests__/fixtures/explore-path-pinning/src/routes/m/projects/[id]/runs/[runId]/+page.ts new file mode 100644 index 0000000..c5fe2b7 --- /dev/null +++ b/__tests__/fixtures/explore-path-pinning/src/routes/m/projects/[id]/runs/[runId]/+page.ts @@ -0,0 +1,67 @@ +/** Mobile run feed — event stream + scroll pinning for the run page. */ + +export interface FeedEvent { + id: string; + kind: 'output' | 'tool' | 'error'; + content: string; +} + +const EVENT_CAP = 300; + +let events: FeedEvent[] = []; +let workingLine: string | null = null; + +/** Whether the reader is at the tail of the feed (within 50px). */ +let feedAtBottom = true; + +interface FeedElement { + scrollTop: number; + scrollHeight: number; + clientHeight: number; +} + +let feedEl: FeedElement | null = null; + +export function bindFeedElement(el: FeedElement | null): void { + feedEl = el; +} + +/** Track the reader's position; called from the feed's scroll listener. */ +export function handleFeedScroll(): void { + const el = feedEl; + if (!el) return; + feedAtBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 50; +} + +/** Re-pin after async content growth (image loads), only when at the tail. */ +export function pinFeedIfNearBottom(): void { + const el = feedEl; + if (!el) return; + if (feedAtBottom) { + el.scrollTop = el.scrollHeight; + } +} + +export function appendEvent(event: FeedEvent): void { + events = events.length >= EVENT_CAP + ? [...events.slice(-(EVENT_CAP - 1)), event] + : [...events, event]; + if (feedAtBottom) { + pinFeedIfNearBottom(); + } +} + +export function setWorkingLine(line: string | null): void { + workingLine = line; + pinFeedIfNearBottom(); +} + +export function resetFeed(): void { + events = []; + workingLine = null; + feedAtBottom = true; +} + +export function feedSnapshot(): { events: FeedEvent[]; workingLine: string | null } { + return { events: [...events], workingLine }; +} diff --git a/__tests__/identifier-segments.test.ts b/__tests__/identifier-segments.test.ts index 11884bf..477c702 100644 --- a/__tests__/identifier-segments.test.ts +++ b/__tests__/identifier-segments.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest'; import { splitIdentifierSegments, extractProseCandidates, + extractSegmentSearchWords, normalizeProseWord, segmentLookupVariants, } from '../src/search/identifier-segments'; @@ -101,3 +102,32 @@ describe('segmentLookupVariants — light plural folding', () => { expect(segmentLookupVariants('boxes')).toEqual(['boxes']); // -es strip would go sub-minimum }); }); + +describe('extractSegmentSearchWords — query words for the search-side vocab supplement', () => { + it('keeps prose words and adds camel-token segments', () => { + const words = extractSegmentSearchWords('auto-scroll to bottom — atBottom tracking'); + // Prose candidates survive as before… + expect(words).toContain('scroll'); + expect(words).toContain('bottom'); + expect(words).toContain('tracking'); + // …and the camel token contributed its ≥4-char segments ("at" is under + // the prose minimum; "bottom" arrives from the split even when the prose + // pass missed it). + expect(extractSegmentSearchWords('where is atBottom set')).toContain('bottom'); + }); + + it('splits multi-hump tokens into every usable segment', () => { + const words = extractSegmentSearchWords('trace pinFeedIfNearBottom please'); + expect(words).toEqual(expect.arrayContaining(['feed', 'near', 'bottom'])); + }); + + it('does not invent segments for plain prose', () => { + const words = extractSegmentSearchWords('how does checkout work'); + expect(words).toContain('checkout'); + expect(words).not.toContain('check'); + }); + + it('returns nothing for an empty query', () => { + expect(extractSegmentSearchWords('')).toEqual([]); + }); +}); diff --git a/__tests__/query-paths.test.ts b/__tests__/query-paths.test.ts new file mode 100644 index 0000000..13a4747 --- /dev/null +++ b/__tests__/query-paths.test.ts @@ -0,0 +1,129 @@ +/** + * File-path recognition in explore queries (src/search/query-paths.ts). + * + * The originating bug: an agent named two SvelteKit route files by exact path + * (`src/routes/m/projects/[id]/runs/[runId]/+page.svelte`) and the explore + * pipeline shredded them — the seeding tokenizer splits on brackets, so the + * fragments `runId`/`scope` seeded as "named symbols" and headlined the blast + * radius, while FTS admitted every sibling `+page.svelte` off the `page`/`runs` + * fragments. These tests pin the module that stops that: path spans resolve + * against the indexed file list, matching files pin, and the spans leave the + * query. Resolution IS the detector — slash-bearing non-paths stay untouched. + */ +import { describe, it, expect } from 'vitest'; +import { extractQueryPaths, queryMightContainPaths } from '../src/search/query-paths'; + +const INDEX = [ + 'src/routes/m/projects/[id]/runs/[runId]/+page.svelte', + 'src/routes/m/projects/[id]/chat/[scope]/+page.svelte', + 'src/routes/m/projects/[id]/+page.svelte', + 'src/routes/(protected)/chat-window/+page.svelte', + 'src/lib/chat-manager.ts', + 'src/lib/task-runner-manager.ts', + 'src/lib/stores/sqlite-store.ts', + 'src/lib/stores/postgresql-store.ts', +]; + +describe('queryMightContainPaths — the cheap pre-gate', () => { + it('fires on slashes and dotted basenames', () => { + expect(queryMightContainPaths('look at src/lib/chat-manager.ts')).toBe(true); + expect(queryMightContainPaths('look at chat-manager.ts please')).toBe(true); + }); + + it('stays quiet on plain prose and Class.method spans', () => { + expect(queryMightContainPaths('how does the scroll pinning work')).toBe(false); + // `.isPackaged` is 10 chars — past the 8-char extension cap. + expect(queryMightContainPaths('what reads app.isPackaged here')).toBe(false); + }); +}); + +describe('extractQueryPaths — resolution and stripping', () => { + it('resolves a bracketed SvelteKit path and strips it from the query', () => { + const q = 'auto-scroll logic in src/routes/m/projects/[id]/runs/[runId]/+page.svelte — atBottom tracking'; + const out = extractQueryPaths(q, INDEX); + expect(out.pinnedFiles).toEqual(['src/routes/m/projects/[id]/runs/[runId]/+page.svelte']); + expect(out.strippedQuery).not.toContain('+page.svelte'); + expect(out.strippedQuery).not.toContain('runId'); + expect(out.strippedQuery).toContain('atBottom tracking'); + expect(out.unresolvedPathSpans).toEqual([]); + }); + + it('pins multiple named files in appearance order', () => { + const q = 'compare src/routes/m/projects/[id]/chat/[scope]/+page.svelte and src/routes/m/projects/[id]/runs/[runId]/+page.svelte'; + const out = extractQueryPaths(q, INDEX); + expect(out.pinnedFiles).toEqual([ + 'src/routes/m/projects/[id]/chat/[scope]/+page.svelte', + 'src/routes/m/projects/[id]/runs/[runId]/+page.svelte', + ]); + }); + + it('resolves a (protected) route-group path — parens are path characters', () => { + const out = extractQueryPaths('read src/routes/(protected)/chat-window/+page.svelte', INDEX); + expect(out.pinnedFiles).toEqual(['src/routes/(protected)/chat-window/+page.svelte']); + }); + + it('resolves an absolute path by walking suffixes to the indexed relative path', () => { + const q = 'fix /Users/colby/dev/beads-live-dashboard/src/lib/chat-manager.ts'; + const out = extractQueryPaths(q, INDEX); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + }); + + it('resolves a unique basename and a partial path', () => { + expect(extractQueryPaths('see chat-manager.ts', INDEX).pinnedFiles) + .toEqual(['src/lib/chat-manager.ts']); + expect(extractQueryPaths('see stores/sqlite-store.ts', INDEX).pinnedFiles) + .toEqual(['src/lib/stores/sqlite-store.ts']); + }); + + it('strips wrapping punctuation and line references', () => { + const out = extractQueryPaths('the bug (see `src/lib/chat-manager.ts:243`).', INDEX); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + const hash = extractQueryPaths('regression at src/lib/task-runner-manager.ts#L88-L120', INDEX); + expect(hash.pinnedFiles).toEqual(['src/lib/task-runner-manager.ts']); + }); + + it('treats an over-ambiguous basename as unresolved — stripped and reported', () => { + const out = extractQueryPaths('why do all +page.svelte files flash', INDEX); + expect(out.pinnedFiles).toEqual([]); + expect(out.unresolvedPathSpans).toEqual(['+page.svelte']); + expect(out.strippedQuery).toBe('why do all files flash'); + }); + + it('strips and reports a clearly-path-shaped span that matches nothing', () => { + const out = extractQueryPaths('crash in src/routes/gone/missing-page.svelte on load', INDEX); + expect(out.pinnedFiles).toEqual([]); + expect(out.unresolvedPathSpans).toEqual(['src/routes/gone/missing-page.svelte']); + expect(out.strippedQuery).toBe('crash in on load'); + }); + + it('leaves slash-bearing non-paths alone', () => { + const q = 'does gen_server:call/2 block and/or timeout'; + const out = extractQueryPaths(q, INDEX); + expect(out.pinnedFiles).toEqual([]); + expect(out.unresolvedPathSpans).toEqual([]); + expect(out.strippedQuery).toBe(q); + }); + + it('dedupes a path named twice and honors maxPins', () => { + const twice = extractQueryPaths( + 'src/lib/chat-manager.ts wraps src/lib/chat-manager.ts', INDEX, + ); + expect(twice.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + + const capped = extractQueryPaths( + 'src/lib/chat-manager.ts src/lib/task-runner-manager.ts', INDEX, { maxPins: 1 }, + ); + expect(capped.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + }); + + it('matches case-insensitively but returns the indexed spelling', () => { + const out = extractQueryPaths('SRC/LIB/CHAT-MANAGER.TS', INDEX); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + }); + + it('passes through untouched when nothing resolves', () => { + const q = 'plain prose question about scrolling'; + const out = extractQueryPaths(q, INDEX); + expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [] }); + }); +}); diff --git a/src/context/index.ts b/src/context/index.ts index ad4d63b..a3da12c 100644 --- a/src/context/index.ts +++ b/src/context/index.ts @@ -171,6 +171,7 @@ const DEFAULT_FIND_OPTIONS: Required = { minScore: 0.3, edgeKinds: [], nodeKinds: HIGH_VALUE_NODE_KINDS, // Filter out imports/exports by default + seedNames: [], // Segment-vocab supplement — filled by the facade }; // Re-export the low-confidence sentinel (defined in a dependency-free leaf so @@ -460,13 +461,37 @@ export class ContextBuilder { // Step 2: Look up exact matches for extracted symbols let exactMatches: SearchResult[] = []; - if (symbolsFromQuery.length > 0) { + if (symbolsFromQuery.length > 0 || opts.seedNames.length > 0) { try { - // Get more results so we can apply co-location boosting before trimming - exactMatches = this.queries.findNodesByExactName(symbolsFromQuery, { - limit: Math.ceil(opts.searchLimit * 5), - kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, - }); + if (symbolsFromQuery.length > 0) { + // Get more results so we can apply co-location boosting before trimming + exactMatches = this.queries.findNodesByExactName(symbolsFromQuery, { + limit: Math.ceil(opts.searchLimit * 5), + kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, + }); + } + + // Step 2a: segment-vocabulary seeds. Word-level query terms cannot + // reach camelCase names through FTS (one token per name), so the + // caller resolves query words → names via the segment vocab and hands + // them in as seedNames. Merged at a dampened score — a symbol the + // query names outright must outrank a segment-derived one — but + // BEFORE the co-location boost below, because several seeds landing + // in one file (pinFeedIfNearBottom + feedAtBottom + handleFeedScroll) + // is exactly the evidence that file is the answer. + if (opts.seedNames.length > 0) { + const seedResults = this.queries.findNodesByExactName(opts.seedNames, { + limit: Math.ceil(opts.searchLimit * 3), + kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, + }); + const known = new Set(exactMatches.map((r) => r.node.id)); + for (const r of seedResults) { + if (known.has(r.node.id)) continue; + known.add(r.node.id); + exactMatches.push({ ...r, score: r.score * 0.6 }); + } + logDebug('Segment seed matches', { seedNames: opts.seedNames, added: known.size }); + } // Co-location boost: when multiple extracted symbols appear in the same file, // those results are much more likely to be what the user is looking for. diff --git a/src/index.ts b/src/index.ts index a05661c..15eaf7a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -54,7 +54,7 @@ import { EXTRACTION_VERSION } from './extraction/extraction-version'; import { getCodeGraphDir } from './directory'; import { deriveProjectNameTokens } from './search/query-utils'; import { CodeGraphPackageVersion } from './mcp/version'; -import { segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments'; +import { extractSegmentSearchWords, segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments'; import { createYielder } from './resolution/cooperative-yield'; import { minRefsForPool } from './resolution/resolver-pool'; @@ -1823,7 +1823,23 @@ export class CodeGraph { query: string, options?: FindRelevantContextOptions ): Promise { - return this.contextBuilder.findRelevantContext(query, options); + // Segment-vocab supplement: FTS keeps camelCase names as single tokens, + // so a word-level query ("auto-scroll to bottom") can never reach + // `pinFeedIfNearBottom` through search alone. Resolve the query's words + // against name_segment_vocab (same precision rules as the prompt hook: + // co-occurrence, else rare singles, verified against live nodes) and hand + // the names down as dampened exact-name seeds. Callers that pass their + // own seedNames keep them; failures degrade to no supplement. + let seedNames = options?.seedNames; + if (seedNames === undefined) { + try { + seedNames = this.getSegmentMatches(extractSegmentSearchWords(query), 8) + .map((m) => m.name); + } catch { + seedNames = []; + } + } + return this.contextBuilder.findRelevantContext(query, { ...options, seedNames }); } /** diff --git a/src/mcp/explore-diagnostics.ts b/src/mcp/explore-diagnostics.ts index 4de9c60..0c654e0 100644 --- a/src/mcp/explore-diagnostics.ts +++ b/src/mcp/explore-diagnostics.ts @@ -60,6 +60,8 @@ export interface ExploreCandidateMeta { graphScore: number; termHits: number; nodes: number; + /** The query named this file by PATH — pinned rank/allocation treatment. */ + pinned?: boolean; named: boolean; central: boolean; entry: boolean; @@ -579,6 +581,7 @@ export class ExploreDiagnostics { graphScore: round6(r.graphScore), termHits: r.termHits, nodes: r.nodes, + pinned: r.pinned ?? false, named: r.named, central: r.central, entry: r.entry, @@ -797,6 +800,7 @@ export function renderTable(report: ExploreDiagnosticReport): string { function flagString(f: ExploreDiagnosticFile): string { const flags: string[] = []; + if (f.pinned) flags.push('pinned'); if (f.named) flags.push('named'); if (f.entry) flags.push('entry'); if (f.central) flags.push('central'); diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index d1d0135..4ad7e64 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -32,6 +32,7 @@ import { import type { PendingFile } from '../sync'; import type { Node, Edge, SearchResult, Subgraph, NodeKind } from '../types'; import { isTestFile, normalizeNameToken } from '../search/query-utils'; +import { extractQueryPaths, queryMightContainPaths } from '../search/query-paths'; import { existsSync, readFileSync, @@ -631,6 +632,13 @@ export interface ExploreAllocationCandidate { worth: number; /** Carries a symbol on the rendered flow spine. */ spine: boolean; + /** + * The query named this file by PATH (see query-paths.ts). Pinned files are + * never cliffed or trimmed, and weigh at least as much as the strongest + * candidate — the agent asked for the file itself, so starving it on text/ + * graph scores (which a pure-path query doesn't produce) defeats the ask. + */ + pinned?: boolean; } export interface ExploreAllocation { @@ -676,7 +684,16 @@ export function allocateExploreBudget( return Number.isFinite(w) ? w : 0; }; - const weights = new Map(candidates.map((c) => [c.path, weightOf(c)])); + // Pinned files weigh at least as much as the strongest raw candidate: their + // score is whatever the stripped query happened to match (for a pure-path + // query, nearly nothing), and a proportional split on that would fund the + // named file worst of all. Floor of 1 covers the all-pinned/zero-score case. + const rawWeights = new Map(candidates.map((c) => [c.path, weightOf(c)])); + const topRaw = Math.max(...rawWeights.values()); + const weights = new Map(candidates.map((c) => [ + c.path, + c.pinned ? Math.max(rawWeights.get(c.path) ?? 0, topRaw, 1) : (rawWeights.get(c.path) ?? 0), + ])); const topWeight = Math.max(...weights.values()); if (!(topWeight > 0)) return empty; @@ -686,7 +703,7 @@ export function allocateExploreBudget( const cliffed: string[] = []; let admitted: ExploreAllocationCandidate[] = []; for (const c of candidates) { - if (!c.spine && (weights.get(c.path) ?? 0) < cliffAt) cliffed.push(c.path); + if (!c.spine && !c.pinned && (weights.get(c.path) ?? 0) < cliffAt) cliffed.push(c.path); else admitted.push(c); } // Never cliff every candidate: an empty response costs a whole round-trip. @@ -705,7 +722,7 @@ export function allocateExploreBudget( if (admitted.length > affordable) { const byWeight = [...admitted].sort((a, b) => (weights.get(b.path) ?? 0) - (weights.get(a.path) ?? 0)); const keep = new Set(byWeight.slice(0, affordable).map((c) => c.path)); - for (const c of admitted) if (c.spine) keep.add(c.path); + for (const c of admitted) if (c.spine || c.pinned) keep.add(c.path); for (const c of admitted) if (!keep.has(c.path)) cliffed.push(c.path); admitted = admitted.filter((c) => keep.has(c.path)); } @@ -3223,6 +3240,34 @@ export class ToolHandler { } const maxFiles = clamp((args.maxFiles as number) || budget.defaultMaxFiles, 1, 20); + // File paths named in the query become PINNED files: guaranteed admission, + // top of the rank order, funded first — and their span is REMOVED from the + // matching query. Runs on the RAW query (normalizeQuerySpelling strips + // `/digits` tails, which would mangle numeric path segments). Without this, + // a SvelteKit path like `runs/[runId]/+page.svelte` was shredded by the + // seeding tokenizer (splits on brackets → `runId` seeded as a "named + // symbol") and by FTS (`page`/`runs` fragments admitted every sibling + // `+page.svelte`), starving the very files the agent asked for. + let pinnedFiles: string[] = []; + let unresolvedPathSpans: string[] = []; + let matchQuery = query; + if (queryMightContainPaths(rawQuery)) { + try { + const extraction = extractQueryPaths( + rawQuery, + cg.getFiles().map((f) => f.path), + { maxPins: maxFiles }, + ); + if (extraction.pinnedFiles.length > 0 || extraction.unresolvedPathSpans.length > 0) { + pinnedFiles = extraction.pinnedFiles; + unresolvedPathSpans = extraction.unresolvedPathSpans; + matchQuery = normalizeQuerySpelling(extraction.strippedQuery); + } + } catch { /* path pinning must never fail an explore call */ } + } + const pinnedSet = new Set(pinnedFiles); + const pinnedOrder = new Map(pinnedFiles.map((p, i) => [p, i])); + // Per-file allocation diagnostic (CG-4). `null` unless CODEGRAPH_EXPLORE_DEBUG // is set — every `diag?.` below is then a no-op and the response is // byte-identical. It only OBSERVES: it must never feed back into rendering. @@ -3279,16 +3324,34 @@ export class ToolHandler { // Use a large maxNodes budget — explore has its own 35k char output limit // that prevents context bloat, so more nodes just means better coverage // across entry points (especially for large files like Svelte components). - const subgraph = await cg.findRelevantContext(query, { + // Matching runs on the path-stripped query; `query` stays for display. + const subgraph = await cg.findRelevantContext(matchQuery, { searchLimit: 8, traversalDepth: 3, maxNodes: 200, minScore: 0.2, }); + // Pinned files' symbols enter the gather unconditionally — the agent named + // the file itself, so its contents ARE the answer regardless of what the + // stripped query text matched (which, for a pure-path query, is nothing). + const PINNED_FILE_NODE_CAP = 300; + for (const fp of pinnedFiles) { + let fileNodes: Node[] = []; + try { fileNodes = cg.getNodesInFile(fp); } catch { continue; } + fileNodes + .filter((n) => n.kind !== 'file' && n.kind !== 'import' && n.kind !== 'export') + .sort((a, b) => a.startLine - b.startLine) + .slice(0, PINNED_FILE_NODE_CAP) + .forEach((n) => { if (!subgraph.nodes.has(n.id)) subgraph.nodes.set(n.id, n); }); + } + if (subgraph.nodes.size === 0) { diag?.finishEmpty('no relevant code found — empty subgraph'); - const empty = `No relevant code found for "${query}"`; + const missNote = unresolvedPathSpans.length > 0 + ? ` (no indexed file uniquely matches ${unresolvedPathSpans.map((s) => `\`${s}\``).join(', ')})` + : ''; + const empty = `No relevant code found for "${query}"${missNote}`; // Still an explore call, so it is still recorded: an empty answer spends a // call against the tier budget even though it emits no source. return this.exploreResult(empty, { @@ -3351,11 +3414,18 @@ export class ToolHandler { { const FILE_EXT = /\.(?:java|kt|kts|ts|tsx|js|jsx|mjs|cjs|cs|py|go|rb|php|swift|rs|cpp|cc|cxx|c|h|hpp|scala|lua|dart|vue|svelte|astro|erl|hrl)$/i; const CALLABLE = new Set(['method', 'function', 'component', 'constructor']); + // Variables/constants seed too: in Svelte/React a `$state` variable + // (`chatAtBottom`, `feedAtBottom`) is exactly the kind of symbol an agent + // names in a query, and the exact-name search channel already returns + // them — only this seeding tier was callable-only. The NL-stopword guard + // below applies unchanged, so bare English words still can't seed a + // same-named local. Callables keep priority via the body-size sort. + const SEEDABLE = new Set([...CALLABLE, 'variable', 'constant']); const isTestPath = (p: string) => /(^|\/)(tests?|specs?|__tests__|testdata|mocks?|fixtures?)\//i.test(p) || /\.(test|spec)\.[a-z]+$/i.test(p); const bodyLines = (n: Node) => Math.max(0, (n.endLine ?? n.startLine) - n.startLine); const callerCount = (n: Node) => { try { return cg.getCallers(n.id).length; } catch { return 0; } }; const tokens = [...new Set( - query.split(/[\s,()[\]]+/) + matchQuery.split(/[\s,()[\]]+/) .map((t) => t.replace(FILE_EXT, '').trim()) .filter((t) => t.length >= 3 && /^[A-Za-z_$][\w$]*(?:(?:::|\.)[\w$]+)*$/.test(t)) )].slice(0, 16); @@ -3430,24 +3500,26 @@ export class ToolHandler { } } let cands = raw - .filter((n) => CALLABLE.has(n.kind) && !isTestPath(n.filePath)) + .filter((n) => SEEDABLE.has(n.kind) && !isTestPath(n.filePath)) .sort((a, b) => (bodyLines(b) > 1 ? 1 : 0) - (bodyLines(a) > 1 ? 1 : 0) || bodyLines(b) - bodyLines(a)); // Field-name seeding fallback (#1196): a camelCase token that names NO // definition of its own is usually an object-literal key / API field // (`profileInfo`) — no node exists, so it contributed zero seeds and // the files that DEFINE it (`getProfileInfoV2` in profileController) - // never surfaced. Seed its camel-infix definers instead: callables - // whose name contains the token at a hump boundary or as a prefix. + // never surfaced. Seed its camel-infix definers instead: seedable + // symbols (callables + variables — `atBottom` must reach the `$state` + // variables `feedAtBottom`/`chatAtBottom`) whose name contains the + // token at a hump boundary or as a prefix. // Exact-empty + camel-shaped only (bare words keep the NL-stopword // guard below), shortest-first, capped so a hot infix can't flood. if (cands.length === 0 && !isQual && /[a-z][A-Z]/.test(t)) { const lcToken = t.toLowerCase(); cands = cg .getNodesByNameSubstring(t, { - kinds: ['function', 'method', 'component'], + kinds: ['function', 'method', 'component', 'variable', 'constant'], limit: 60, }) - .filter((n) => CALLABLE.has(n.kind) && !isTestPath(n.filePath)) + .filter((n) => SEEDABLE.has(n.kind) && !isTestPath(n.filePath)) .filter((n) => { const idx = n.name.toLowerCase().indexOf(lcToken); if (idx < 0) return false; @@ -3624,8 +3696,9 @@ export class ToolHandler { fileGroups.set(node.filePath, group); } - // Extract query terms for relevance checking - const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length >= 3); + // Extract query terms for relevance checking (path-stripped: a pinned + // file's own path fragments must not count as "term hits" everywhere) + const queryTerms = matchQuery.toLowerCase().split(/\s+/).filter(t => t.length >= 3); // Test/spec/icon/i18n file detector — used by the pre-floor hard filter, the // rank penalty, and the comparator deprioritization. @@ -3715,9 +3788,10 @@ export class ToolHandler { // keep-minimum then pulled two test files back in as the "spread". let candidateFiles = [...fileGroups.entries()]; { - const queryMentionsTests = /\b(test|tests|testing|spec|verify|verifies)\b/i.test(query); + const queryMentionsTests = /\b(test|tests|testing|spec|verify|verifies)\b/i.test(matchQuery); if (!queryMentionsTests) { - const nonLow = candidateFiles.filter(([p]) => !isLowValue(p)); + // A pinned file is exempt: naming a test file by path IS asking for it. + const nonLow = candidateFiles.filter(([p]) => !isLowValue(p) || pinnedSet.has(p)); if (nonLow.length >= 2) { candidateFiles = nonLow; } @@ -3732,7 +3806,9 @@ export class ToolHandler { SCORE_FLOOR_ABSOLUTE, Math.min(SCORE_FLOOR_MAX, topScore * SCORE_FLOOR_FRACTION_OF_TOP), ); - let relevantFiles = candidateFiles.filter(([, group]) => group.score >= scoreFloor); + let relevantFiles = candidateFiles.filter( + ([fp, group]) => group.score >= scoreFloor || pinnedSet.has(fp), + ); if (relevantFiles.length < SCORE_FLOOR_KEEP_MIN) { // Backfill from what the RELATIVE floor cut, best first, at two strengths: // @@ -3747,8 +3823,11 @@ export class ToolHandler { // worst outcome on the board — the agent falls straight back to grep. const minEvidence = relevantFiles.length === 0 ? Number.EPSILON : SCORE_FLOOR_ABSOLUTE; relevantFiles = candidateFiles - .filter(([, group]) => group.score >= minEvidence) - .sort((a, b) => b[1].score - a[1].score || b[1].nodes.length - a[1].nodes.length) + .filter(([fp, group]) => group.score >= minEvidence || pinnedSet.has(fp)) + .sort((a, b) => + (pinnedSet.has(b[0]) ? 1 : 0) - (pinnedSet.has(a[0]) ? 1 : 0) + || b[1].score - a[1].score + || b[1].nodes.length - a[1].nodes.length) .slice(0, Math.max(SCORE_FLOOR_KEEP_MIN, relevantFiles.length)); } diag?.setScoreFloor(scoreFloor, relevantFiles.length); @@ -3852,7 +3931,8 @@ export class ToolHandler { // never prunes below 2. if (maxGraph > 0) { const gated = relevantFiles.filter(([fp]) => - (fileGraphScore.get(fp) ?? 0) >= maxGraph * 0.06 + pinnedSet.has(fp) + || (fileGraphScore.get(fp) ?? 0) >= maxGraph * 0.06 || centralFiles.has(fp) || entryFiles.has(fp) || changeSurfaceFiles.has(fp) @@ -3908,7 +3988,15 @@ export class ToolHandler { const aPath = a[0].toLowerCase(); const bPath = b[0].toLowerCase(); - // Agent-named files first (it asked for a symbol defined here by name). + // Pinned files first of all — the agent named the FILE by path, which is + // even more explicit than naming a symbol in it. Among pins, keep the + // order they appeared in the query. + const aPin = pinnedSet.has(a[0]) ? 1 : 0; + const bPin = pinnedSet.has(b[0]) ? 1 : 0; + if (aPin !== bPin) return bPin - aPin; + if (aPin && bPin) return (pinnedOrder.get(a[0]) ?? 0) - (pinnedOrder.get(b[0]) ?? 0); + + // Agent-named files next (it asked for a symbol defined here by name). const aNamed = namedSeedFiles.has(a[0]) ? 1 : 0; const bNamed = namedSeedFiles.has(b[0]) ? 1 : 0; if (aNamed !== bNamed) return bNamed - aNamed; @@ -4010,7 +4098,7 @@ export class ToolHandler { // Compute the flow spine once — used both to prepend the Flow section (below) // and to gate adaptive source sizing: files on the spine get full source, // off-spine peers skeletonize. - const flow = this.buildFlowFromNamedSymbols(cg, query); + const flow = this.buildFlowFromNamedSymbols(cg, matchQuery); // Snapshot every ranked candidate's scoring inputs, in final sort order, so // the diagnostic can show what each file's share of the envelope was BOUGHT @@ -4031,6 +4119,7 @@ export class ToolHandler { graphScore: fileGraphScore.get(fp) ?? 0, termHits: fileTermHits.get(fp) ?? 0, nodes: group.nodes.length, + pinned: pinnedSet.has(fp), named: namedSeedFiles.has(fp), central: centralFiles.has(fp), entry: entryFiles.has(fp), @@ -4051,8 +4140,11 @@ export class ToolHandler { sortedFiles.map(([fp, group]) => ({ path: fp, score: group.score, - worth: rankPenalty(fp), + // A pinned file's bytes are worth full price by definition — the agent + // asked for the file itself, generated/test or not. + worth: pinnedSet.has(fp) ? 1 : rankPenalty(fp), spine: group.nodes.some((n) => flow.pathNodeIds.has(n.id)), + pinned: pinnedSet.has(fp), })), budget, maxFiles, @@ -5804,9 +5896,19 @@ export class ToolHandler { g.nodes.filter((n) => n.kind !== 'import' && n.kind !== 'export').map((n) => n.id), ).size; }, 0); - const summaryLine = survivors.length > 0 + let summaryLine = survivors.length > 0 ? `Found ${shownSymbols} symbol${shownSymbols === 1 ? '' : 's'} across ${survivors.length} file${survivors.length === 1 ? '' : 's'}.` : `Found ${subgraph.nodes.size} symbol${subgraph.nodes.size === 1 ? '' : 's'} across ${fileGroups.size} file${fileGroups.size === 1 ? '' : 's'}.`; + // Path pinning is visible, not silent: say which query-named files were + // honored, and which path spans matched nothing so the agent can correct + // them instead of trusting a response that quietly ignored the path. + const pinnedShown = pinnedFiles.filter((fp) => survivors.includes(fp)).length; + if (pinnedShown > 0) { + summaryLine += ` ${pinnedShown} file${pinnedShown === 1 ? '' : 's'} pinned from the query.`; + } + if (unresolvedPathSpans.length > 0) { + summaryLine += ` No indexed file uniquely matches ${unresolvedPathSpans.map((s) => `\`${s}\``).join(', ')}.`; + } finalText = finalText.replace(SUMMARY_SENTINEL, summaryLine); // Emit the allocation diagnostic from the FINAL text, so per-file bytes and diff --git a/src/search/identifier-segments.ts b/src/search/identifier-segments.ts index 110a6b7..75bfde4 100644 --- a/src/search/identifier-segments.ts +++ b/src/search/identifier-segments.ts @@ -126,6 +126,33 @@ export function extractProseCandidates(prompt: string): string[] { return [...seen]; } +/** + * Words to look up in the segment vocabulary for a SEARCH query (as opposed + * to a prompt-hook gate): the query's prose candidates PLUS the segments of + * its identifier-shaped tokens. An agent's query names concepts both ways — + * "auto-scroll to bottom" (prose) and "atBottom tracking" (camel) — and the + * camel token must still reach the segment "bottom" even though the whole + * token matches no name. Same stopword/length rules as the hook path, since + * both feeds run through {@link extractProseCandidates}. + */ +export function extractSegmentSearchWords(query: string): string[] { + if (!query) return []; + const words = new Set(extractProseCandidates(query)); + const segments: string[] = []; + for (const run of query.match(/[\p{L}\p{N}]+/gu) ?? []) { + // Only camel-humped tokens contribute segments — a plain word's + // "segments" are itself (already covered above), and snake_case arrives + // as separate runs because `_` is not a letter. + if (/[\p{Ll}\p{N}]\p{Lu}/u.test(run)) { + segments.push(...splitIdentifierSegments(run)); + } + } + if (segments.length > 0) { + for (const w of extractProseCandidates(segments.join(' '))) words.add(w); + } + return [...words]; +} + /** * Lookup variants for a prose word: the word itself plus light plural folding * ("services" → service, "dependencies" → dependencie/dependency is NOT diff --git a/src/search/query-paths.ts b/src/search/query-paths.ts new file mode 100644 index 0000000..ce186c8 --- /dev/null +++ b/src/search/query-paths.ts @@ -0,0 +1,213 @@ +/** + * File-path recognition for explore queries. + * + * Agents routinely name files by path in a `codegraph_explore` query — + * "the scroll logic in src/routes/m/projects/[id]/runs/[runId]/+page.svelte" — + * and until this module existed those spans were SHREDDED by the downstream + * tokenizers instead of being read as file references: + * + * - the named-symbol seeder splits on `[\s,()[\]]+`, so SvelteKit/Next + * bracketed segments (`[id]`, `[runId]`) and route groups (`(protected)`) + * exploded the path into fragments; the identifier-shaped survivors + * (`runId`, `scope`) then seeded as "symbols the agent named" and + * headlined the blast radius; + * - FTS saw the fragments (`page`, `chat`, `runs`) and admitted every + * sibling `+page.svelte` in the repo, which ate the output envelope and + * truncated the files the agent actually asked for. + * + * `extractQueryPaths` finds path-like spans, resolves them against the + * INDEXED file list (resolution IS the detector — `and/or`, `gen_server:call/2` + * and other slash-bearing non-paths match nothing and are left alone), and + * returns the matches as pinned files plus the query with those spans removed. + * Callers treat pinned files as first-class: guaranteed admission, top rank, + * funded first. Pure string work — no DB, no fs — so it is trivially testable + * and safe inside the query-pool workers. + */ + +export interface QueryPathExtraction { + /** The query with resolved/clearly-path spans removed, whitespace-joined. */ + strippedQuery: string; + /** Indexed file paths the query named, appearance-ordered, deduped. */ + pinnedFiles: string[]; + /** + * Spans that are unambiguously path-shaped but resolved to nothing (stale + * path, unindexed file) or to too many files (bare `+page.svelte`). Stripped + * from the query — their fragments could only mint junk matches — and + * surfaced to the agent so the miss is visible instead of silent. + */ + unresolvedPathSpans: string[]; +} + +/** + * Cheap pre-gate so callers only fetch the indexed file list when the query + * could possibly contain a path: a slash, or a dot-extension-shaped tail + * (`chat-manager.ts`). Extensions cap at 8 chars, which keeps `Class.method` + * spans (`app.isPackaged`) from qualifying. + */ +export function queryMightContainPaths(query: string): boolean { + return /[/\\]/.test(query) || /\.[A-Za-z][A-Za-z0-9]{0,7}(?=[\s,;:)\]'"`]|$)/.test(query); +} + +/** + * Longest span→suffix walk tried per span. 8 covers an absolute macOS path + * (`/Users//dev//…`) over a deeply nested repo-relative file; + * deeper prefixes buy nothing. + */ +const MAX_SUFFIX_TRIES = 8; +/** Spans examined per query — a prose sentence is not 50 paths. */ +const MAX_CANDIDATE_SPANS = 8; + +/** `name.ext` shape with a plausible source extension (no slash required). */ +const DOTTED_BASENAME = /^[^\s/\\]+\.[A-Za-z][A-Za-z0-9]{0,7}$/; + +/** + * Strip prose punctuation wrapped around a token without eating punctuation + * that is PART of the path: quotes/backticks always strip; a trailing `)`/`]` + * strips only when the token has no matching opener (so `(protected)` and + * `[id]` segments survive, while "…(see src/foo.ts)" loses its parenthesis); + * a leading `(`/`[` mirrors that. Trailing sentence punctuation strips last, + * so "src/foo.ts." resolves. + */ +function stripWrapping(token: string): string { + let s = token; + for (;;) { + const first = s[0]; + if (!first) break; + if ('\'"`<'.includes(first)) { s = s.slice(1); continue; } + if (first === '(' && !s.includes(')')) { s = s.slice(1); continue; } + if (first === '[' && !s.includes(']')) { s = s.slice(1); continue; } + if (first === '{' && !s.includes('}')) { s = s.slice(1); continue; } + break; + } + for (;;) { + const last = s[s.length - 1]; + if (!last) break; + if ('\'"`>.,;!?'.includes(last)) { s = s.slice(0, -1); continue; } + if (last === ')' && !s.includes('(')) { s = s.slice(0, -1); continue; } + if (last === ']' && !s.includes('[')) { s = s.slice(0, -1); continue; } + if (last === '}' && !s.includes('{')) { s = s.slice(0, -1); continue; } + break; + } + // Line references ride along in agent-written paths: `foo.ts:123`, + // `foo.ts:12-40`, `foo.ts#L88`. The file is what gets pinned. + s = s.replace(/(?::\d+(?:-\d+)?|#L\d+(?:-L?\d+)?)$/, ''); + return s; +} + +/** Normalize a span into the repo-relative shape the files table stores. */ +function normalizeSpan(span: string): string { + return span + .replace(/\\/g, '/') + .replace(/^(?:\.\/)+/, '') + .replace(/\/{2,}/g, '/') + .replace(/\/+$/, ''); +} + +/** Path-shaped beyond doubt: ≥2 segments and a dot-extension on the last. */ +function isClearlyPathShaped(normalized: string): boolean { + const slash = normalized.lastIndexOf('/'); + if (slash <= 0) return false; + return DOTTED_BASENAME.test(normalized.slice(slash + 1)); +} + +/** + * Resolve one normalized span against the indexed paths: exact match first, + * then segment-aligned suffix matches, dropping leading segments one at a + * time (so an absolute path, or one prefixed with the repo directory name, + * still lands on the indexed repo-relative file). Suffixes only get shorter — + * and therefore only match MORE — so the walk stops at the first suffix that + * matches anything: within budget it resolves, over budget it is ambiguous. + */ +function resolveSpan( + normalizedLower: string, + lowerToOriginal: ReadonlyMap, + maxMatches: number, +): { matches: string[]; ambiguous: boolean } { + const exact = lowerToOriginal.get(normalizedLower); + if (exact) return { matches: [exact], ambiguous: false }; + + const segments = normalizedLower.split('/').filter(Boolean); + const tries = Math.min(segments.length, MAX_SUFFIX_TRIES); + for (let drop = 0; drop < tries; drop++) { + const suffix = segments.slice(drop).join('/'); + if (!suffix) break; + const withSlash = '/' + suffix; + const matches: string[] = []; + for (const [lower, original] of lowerToOriginal) { + if (lower === suffix || lower.endsWith(withSlash)) { + matches.push(original); + if (matches.length > maxMatches) return { matches: [], ambiguous: true }; + } + } + if (matches.length > 0) return { matches, ambiguous: false }; + } + return { matches: [], ambiguous: false }; +} + +export function extractQueryPaths( + query: string, + indexedPaths: readonly string[], + opts: { maxPins?: number; maxMatchesPerSpan?: number } = {}, +): QueryPathExtraction { + const maxPins = Math.max(1, opts.maxPins ?? 8); + const maxMatchesPerSpan = Math.max(1, opts.maxMatchesPerSpan ?? 3); + + const passthrough: QueryPathExtraction = { + strippedQuery: query, + pinnedFiles: [], + unresolvedPathSpans: [], + }; + if (!query.trim() || indexedPaths.length === 0) return passthrough; + + // Lowercase view of the index, built once per call. Last writer wins on a + // case-colliding pair, which is the existing file-view behavior too. + const lowerToOriginal = new Map(); + for (const p of indexedPaths) lowerToOriginal.set(p.toLowerCase(), p); + + const tokens = query.split(/\s+/).filter(Boolean); + const consumed = new Set(); + const pinned: string[] = []; + const pinnedSeen = new Set(); + const unresolved: string[] = []; + let candidatesExamined = 0; + + for (let i = 0; i < tokens.length; i++) { + if (pinned.length >= maxPins) break; + if (candidatesExamined >= MAX_CANDIDATE_SPANS) break; + const stripped = stripWrapping(tokens[i]!); + if (stripped.length < 4) continue; + const hasSlash = /[/\\]/.test(stripped); + if (!hasSlash && !DOTTED_BASENAME.test(stripped)) continue; + + const normalized = normalizeSpan(stripped); + if (!normalized) continue; + candidatesExamined++; + + const { matches, ambiguous } = resolveSpan( + normalized.toLowerCase(), lowerToOriginal, maxMatchesPerSpan, + ); + if (matches.length > 0) { + consumed.add(i); + for (const m of matches) { + if (pinnedSeen.has(m) || pinned.length >= maxPins) continue; + pinnedSeen.add(m); + pinned.push(m); + } + } else if (ambiguous || isClearlyPathShaped(normalized)) { + // A real path that didn't resolve to a usable set. Keeping it in the + // query is strictly worse — its fragments are what minted the junk + // matches this module exists to stop — so strip it and say so. + consumed.add(i); + if (unresolved.length < 4) unresolved.push(normalized); + } + // Anything else (`and/or`, `call/2`, `foo.Bar`) is not a path reference: + // leave the token for the normal matching pipeline. + } + + if (consumed.size === 0) return passthrough; + return { + strippedQuery: tokens.filter((_, i) => !consumed.has(i)).join(' '), + pinnedFiles: pinned, + unresolvedPathSpans: unresolved, + }; +} diff --git a/src/types.ts b/src/types.ts index b0ebfe4..186f57a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -688,4 +688,13 @@ export interface FindRelevantContextOptions { /** Node types to include */ nodeKinds?: NodeKind[]; + + /** + * Extra symbol names to merge in as exact-name search candidates, at a + * dampened score. Fed by the segment-vocabulary supplement (CodeGraph. + * findRelevantContext): word-level query terms can't reach camelCase names + * through FTS — `pinFeedIfNearBottom` is one FTS token — so names whose + * SEGMENTS the query's words name are seeded here instead. + */ + seedNames?: string[]; }