perf(resolution): drop non-unique edge indexes during the bulk resolution window, byte-identical graphs (#1322)

The resolution persist's measured cost is B-tree maintenance on the edges
table's five indexes (offline replay of a 224k-edge resolution set: 2.8s with
all indexes, 1.1s with only the unique identity index, +0.3s to recreate the
rest). On big runs (same >=150k-ref gate as the resolver pool) the four
non-unique edge indexes are now dropped for the batch loop and recreated in
one pass each before synthesis.

Why this is safe:
- idx_edges_identity stays: INSERT OR IGNORE's dedup conflicts on it (#1034),
  and its leftmost column is `source`, so the only mid-window edge reads —
  resolution's supertype walks (implements/extends by source) — keep an index
  via its prefix (verified with EXPLAIN QUERY PLAN).
- The window closes BEFORE synthesis, whose passes read kind-keyed, and on
  every error path (finally).
- A crash inside the window heals on the next DatabaseConnection open —
  schema.sql re-applies CREATE INDEX IF NOT EXISTS, same recovery as the FTS
  bulk-load pattern this mirrors.
- Concurrent readers (a daemon serving the project mid-index) stay correct;
  target/kind-keyed reads degrade to scans only for the window's duration.

dubbo (4,402 files): persists 4.0s -> 3.0s, fresh init 11.9s -> ~11.1s,
graph byte-identical. excalidraw (below the gate): untouched, byte-identical.
Recreation cost ~250ms, logged under CODEGRAPH_SYNTH_TIMINGS as
edge-index-recreate. Suite green (2444).

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
2026-07-16 18:26:30 -05:00
committed by GitHub
co-authored by Claude Fable 5
parent cf38ef65af
commit 567b4ad4be
4 changed files with 85 additions and 2 deletions
+45
View File
@@ -151,6 +151,51 @@ export class DatabaseConnection {
this.recreateFtsTriggers();
}
/**
* Names of the NON-UNIQUE edge indexes dropped for a bulk edge load.
* idx_edges_identity deliberately stays: INSERT OR IGNORE's dedup conflicts
* on it (#1034), and its leftmost column is `source`, so the source-keyed
* reads resolution makes mid-window (supertype walks over
* `implements`/`extends`) keep an index via its prefix — verified with
* EXPLAIN QUERY PLAN. Target-keyed and kind-keyed reads (traversal,
* synthesis) happen only after endBulkEdgeLoad().
*/
private static readonly BULK_EDGE_INDEX_NAMES = [
'idx_edges_kind',
'idx_edges_source_kind',
'idx_edges_target_kind',
'idx_edges_provenance',
] as const;
/**
* Enter bulk-edge-load mode: drop the non-unique edge indexes so the mass
* INSERT OR IGNORE stream pays one B-tree (the identity index) instead of
* five — measured 2.8s → 1.1s inserting a 224k-edge resolution set, with
* recreation costing ~0.3s. MUST be paired with endBulkEdgeLoad(); a crash
* inside the window is healed on the next DatabaseConnection open (schema.sql
* re-applies CREATE INDEX IF NOT EXISTS).
*/
beginBulkEdgeLoad(): void {
for (const idx of DatabaseConnection.BULK_EDGE_INDEX_NAMES) {
this.db.exec(`DROP INDEX IF EXISTS ${idx}`);
}
}
/**
* Leave bulk-edge-load mode: recreate the dropped indexes in one pass each
* over the (now fully loaded) edges table — far cheaper than maintaining
* them per-insert. DDL is extracted from schema.sql so it cannot drift.
*/
endBulkEdgeLoad(): void {
const schemaPath = path.join(__dirname, 'schema.sql');
const schema = fs.readFileSync(schemaPath, 'utf-8');
for (const idx of DatabaseConnection.BULK_EDGE_INDEX_NAMES) {
const m = schema.match(new RegExp(`CREATE INDEX IF NOT EXISTS ${idx}\\b[^;]*;`));
if (!m) throw new Error(`schema.sql: edge index ${idx} not found for bulk-load recreation`);
this.db.exec(m[0]);
}
}
/** Recreate the FTS triggers + rebuild if a bulk-load window never closed. */
private healBulkNodeLoad(): void {
const row = this.db
+9
View File
@@ -1149,6 +1149,15 @@ export class CodeGraph {
): Promise<ResolutionResult> {
return this.resolver.resolveAndPersistBatched(onProgress, undefined, onSynthesisProgress, {
dbPath: this.db.getPath(),
// Bulk-edge-load hooks: on big runs the resolver drops the non-unique
// edge indexes for the batch loop and recreates them before synthesis
// (which reads kind-keyed). Concurrent readers (a daemon serving this
// project mid-index) stay CORRECT during the window — target/kind reads
// just degrade to scans until the recreate.
bulkEdgeLoad: {
begin: () => this.db.beginBulkEdgeLoad(),
end: () => this.db.endBulkEdgeLoad(),
},
});
}
+30 -2
View File
@@ -1336,8 +1336,10 @@ export class ReferenceResolver {
// When provided, big batches fan out across a read-only resolver-worker
// pool with results admitted in canonical order (see resolver-pool.ts).
// Sequential fallback on any pool failure. CODEGRAPH_NO_PARALLEL_RESOLVE=1
// disables entirely.
parallel?: { dbPath: string }
// disables entirely. bulkEdgeLoad hooks (when provided) bracket the batch
// loop with drop/recreate of the non-unique edge indexes on big runs —
// see DatabaseConnection.beginBulkEdgeLoad.
parallel?: { dbPath: string; bulkEdgeLoad?: { begin: () => void; end: () => void } }
): Promise<ResolutionResult> {
// Resolution runs on the indexer's MAIN thread, and the #850 liveness
// watchdog SIGKILLs a process whose event loop stalls past its window (60s
@@ -1444,6 +1446,22 @@ export class ReferenceResolver {
return this.resolveBatchYielding(batch, maybeYield);
};
// Bulk edge load: on big runs, drop the non-unique edge indexes for the
// duration of the batch loop (the identity index stays — OR IGNORE dedup
// and the source-keyed supertype-walk reads both live on it). Recreated in
// the inner finally BEFORE synthesis, whose passes read kind-keyed.
// Measured on a 224k-edge resolution set: insert 2.8s → 1.1s + 0.3s
// recreate. Same ref-count gate as the pool so small syncs never pay the
// recreate cost.
let bulkEdgesActive = false;
if (parallel?.bulkEdgeLoad && total >= minRefsForPool()) {
try {
parallel.bulkEdgeLoad.begin();
bulkEdgesActive = true;
} catch { /* keep the indexes; inserts just pay the per-row maintenance */ }
}
try {
try {
let batch = this.queries.getUnresolvedReferencesBatch(0, batchSize);
let inFlight: InFlight | null = batch.length > 0 ? beginBatch(batch) : null;
@@ -1559,6 +1577,16 @@ export class ReferenceResolver {
batch = nextBatch;
inFlight = nextInFlight;
}
} finally {
// Recreate the edge indexes BEFORE synthesis (kind-keyed reads) and on
// any error path. A crash before this line is healed by the next
// DatabaseConnection open (schema.sql re-applies IF NOT EXISTS).
if (bulkEdgesActive) {
const tIdx = Date.now();
parallel!.bulkEdgeLoad!.end();
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] edge-index-recreate: ${Date.now() - tIdx}ms`);
}
}
// Dynamic-edge synthesis: now that all base `calls` edges are persisted,
// synthesize observer/callback dispatch edges (dispatcher → registered