diff --git a/docs/benchmarks/index-drift-cg33.md b/docs/benchmarks/index-drift-cg33.md new file mode 100644 index 0000000..548e793 --- /dev/null +++ b/docs/benchmarks/index-drift-cg33.md @@ -0,0 +1,103 @@ +# Index drift: incremental sync vs. full rebuild (CG-33) + +Measured 2026-08-06. A live, auto-sync-maintained index **does not converge** to +a clean full rebuild of the identical working tree. On codegraph's own repo, +**4.3% of distinct edges were wrong**, in both directions, overwhelmingly +`calls` edges. + +This matters because it is silent: nothing warns, nothing surfaces it, and the +README tells users the index is never stale and there is nothing to re-run. +Retrieval quality decays invisibly, and the user-visible symptom — an agent +falling back to Read — reads as "codegraph isn't very good" rather than "this +index needs rebuilding." + +## Result + +Subject: codegraph's own `.codegraph/codegraph.db`, long-lived and +incrementally synced, against a full rebuild of the same tree with the same +build. Edges compared as distinct `(source, target, kind)` triples. + +| | count | +|---|---| +| distinct edge triples (rebuild) | 28,809 | +| in rebuild but **missing** from live | **751** | +| in live but **absent** from rebuild (stale) | **476** | +| **total divergent** | **1,227 — 4.3%** | + +Missing edges by kind: `calls=635`, `contains=38`, `references=34`, +`instantiates=21`, `imports=13`, `extends=10`. + +### Raw counts hide it + +Raw edge **rows** were 39,845 live vs 40,122 rebuilt — a benign-looking +0.7%. +The divergence is bidirectional, so a net-count check nets it out and reports +almost nothing wrong. **Any drift detector must compare edge sets, not totals.** + +### The indexer is deterministic + +Control, rebuild vs rebuild on the same tree and build: **0 differing edges** +(28,809 both runs). So the live-vs-rebuild delta is not run-to-run noise. + +### It is resolution, not residue + +Node sets are identical — `files` 501 = 501, `nodes` 10,110 = 10,110, +heuristic edges 36 = 36 — and every integrity check is 0 on *both* indexes: +no duplicate nodes, no orphan edges, no nodes referencing a missing file row. + +Nothing accumulates. Cross-file **resolution** goes stale. + +## Likely mechanism + +`ReferenceResolver` resolves calls and imports by name-matching and the import +graph across the **whole** project. Incremental sync re-parses and re-resolves +only the changed file, so: + +- edges from *other* files into changed symbols are never recomputed → stale + edges retained (the 476); +- edges that should newly form from unchanged files into changed symbols are + never created → missing edges (the 751). + +Start in `src/sync/` and `src/resolution/` — specifically what scope is +re-resolved on a single-file change. + +## Why it degrades retrieval + +Graph mass (RWR) is **relative and normalized**, so call edges missing elsewhere +inflate an unaffected file's share of the mass. Explore ranks files by that mass +(`allocateExploreBudget` weights on it), so drift silently promotes files that +should rank low. + +Observed on a private application repo under heavy development: a generated +ambient-types file carried graph mass **0.24750** on the drifted index vs +**0.13119** on a clean rebuild (~1.9×), and score **49.0** vs **27.0**. On the +drifted index it took **60.7%** of an explore envelope and starved the file the +agent had actually named by symbol, which rendered **251 chars of a 10,970 +reservation**. After a full re-index — no code change — the same query answers +correctly. That incident is what prompted this measurement; see CG-24. + +Severity scales with churn and index age. codegraph's own repo shows 4.3%; +a repo under heavier active development plausibly drifts further. + +## Reproducing + +`scripts/agent-eval/diff-index-drift.mjs` is read-only and diffs two indexes. +Snapshot the live index **before** rebuilding — the original artifact for this +investigation was destroyed by re-indexing over it: + +```bash +cp .codegraph/codegraph.db /tmp/live.db # snapshot FIRST +node dist/bin/codegraph.js index . # full rebuild +node scripts/agent-eval/diff-index-drift.mjs /tmp/live.db .codegraph/codegraph.db +``` + +Exit code is 0 when converged, 1 when drifted. To re-confirm determinism, diff +two consecutive rebuilds — that must report 0. + +## Note on probing an index + +The index file is `.codegraph/codegraph.db`. There is no `graph.db`. `sqlite3` +against a mistyped path **creates an empty database** rather than failing, and +every subsequent query then answers from an empty schema — which reads exactly +like a stale pre-migration index. That produced a wrong root cause during this +investigation. `diff-index-drift.mjs` checks `existsSync` before opening for +exactly this reason. diff --git a/scripts/agent-eval/diff-index-drift.mjs b/scripts/agent-eval/diff-index-drift.mjs new file mode 100644 index 0000000..9459549 --- /dev/null +++ b/scripts/agent-eval/diff-index-drift.mjs @@ -0,0 +1,102 @@ +#!/usr/bin/env node +/** + * Diff two CodeGraph indexes of the SAME tree — typically a live, + * incrementally-synced `.codegraph/codegraph.db` against a clean full rebuild + * of the identical working tree (CG-33). + * + * Non-destructive: it only reads. Rebuilding is the caller's job, so the live + * index is never clobbered by the tool measuring it — the mistake that cost the + * original CG-33 artifact. + * + * # snapshot the live index BEFORE touching it + * cp .codegraph/codegraph.db /tmp/live.db + * node dist/bin/codegraph.js index . + * node scripts/agent-eval/diff-index-drift.mjs /tmp/live.db .codegraph/codegraph.db + * + * Edges are compared as distinct `(source, target, kind)` triples. Raw row + * counts are NOT a drift signal: a bidirectional divergence nets out. On the + * codegraph repo the raw counts differed by +0.7% while 4.3% of distinct edges + * were actually wrong. + */ +import { DatabaseSync } from 'node:sqlite'; +import { existsSync } from 'node:fs'; + +const [livePath, rebuiltPath] = process.argv.slice(2); +if (!livePath || !rebuiltPath) { + console.error('usage: diff-index-drift.mjs '); + process.exit(2); +} +for (const p of [livePath, rebuiltPath]) { + if (!existsSync(p)) { + // node:sqlite CREATES a missing file rather than failing, which silently + // yields an empty schema and a confident, wrong conclusion. Refuse first. + console.error(`not found: ${p}`); + process.exit(2); + } +} + +const open = (p) => new DatabaseSync(p, { readOnly: true }); +const live = open(livePath); +const rebuilt = open(rebuiltPath); + +const scalar = (db, q) => db.prepare(q).get().n; +const edgeKey = (r) => `${r.source}\u0000${r.target}\u0000${r.kind}`; + +const liveEdges = live.prepare('select source, target, kind from edges').all(); +const rebuiltEdges = rebuilt.prepare('select source, target, kind from edges').all(); +const liveSet = new Set(liveEdges.map(edgeKey)); +const rebuiltSet = new Set(rebuiltEdges.map(edgeKey)); + +const missing = rebuiltEdges.filter((r) => !liveSet.has(edgeKey(r))); // should exist, doesn't +const stale = liveEdges.filter((r) => !rebuiltSet.has(edgeKey(r))); // exists, shouldn't +const divergent = missing.length + stale.length; + +const byKind = (rows) => { + const m = new Map(); + for (const r of rows) m.set(r.kind, (m.get(r.kind) ?? 0) + 1); + return [...m].sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}=${v}`).join(', ') || '(none)'; +}; + +const pct = (n, d) => (d ? ((n / d) * 100).toFixed(1) : '0.0'); + +console.log(`live ${livePath}`); +console.log(`rebuilt ${rebuiltPath}`); +console.log(''); +console.log('counts live rebuilt'); +for (const [label, q] of [ + ['files', 'select count(*) n from files'], + ['nodes', 'select count(*) n from nodes'], + ['edges (rows)', 'select count(*) n from edges'], + // Grouped rather than `count(distinct a || b || c)`: bare concatenation has no + // separator, so `(ab, c)` and `(a, bc)` would collapse into one. + ['edges (distinct)', 'select count(*) n from (select distinct source, target, kind from edges)'], + ['heuristic edges', "select count(*) n from edges where provenance='heuristic'"], +]) { + console.log(` ${label.padEnd(20)} ${String(scalar(live, q)).padEnd(9)} ${scalar(rebuilt, q)}`); +} + +console.log(''); +console.log('edge divergence (distinct triples)'); +console.log(` missing from live: ${missing.length} — ${byKind(missing)}`); +console.log(` stale in live: ${stale.length} — ${byKind(stale)}`); +console.log(` TOTAL divergent: ${divergent} (${pct(divergent, rebuiltSet.size)}% of ${rebuiltSet.size})`); + +// Integrity checks — these separate "resolution went stale" (edges wrong, nodes +// identical) from "residue accumulated" (duplicate/orphan rows). CG-33 is the +// former: on the codegraph repo every check below was 0 on BOTH indexes. +console.log(''); +console.log('integrity live rebuilt'); +for (const [label, q] of [ + ['duplicate nodes', 'select count(*) n from (select file_path,name,kind,start_line from nodes group by 1,2,3,4 having count(*)>1)'], + ['orphan edges', 'select count(*) n from edges e where not exists(select 1 from nodes where id=e.source) or not exists(select 1 from nodes where id=e.target)'], + ['nodes w/ missing file row', 'select count(*) n from nodes nd where not exists(select 1 from files f where f.path=nd.file_path)'], +]) { + console.log(` ${label.padEnd(30)} ${String(scalar(live, q)).padEnd(6)} ${scalar(rebuilt, q)}`); +} + +console.log(''); +console.log(divergent === 0 + ? 'CONVERGED — the synced index matches a full rebuild.' + : `DRIFTED — ${divergent} edges differ. Rebuild-vs-rebuild is 0 (the indexer is deterministic), so this is sync divergence, not noise.`); + +process.exitCode = divergent === 0 ? 0 : 1;