perf(resolution): drop non-unique edge indexes during the bulk resolution window, byte-identical graphs (#1322)
The resolution persist's measured cost is B-tree maintenance on the edges table's five indexes (offline replay of a 224k-edge resolution set: 2.8s with all indexes, 1.1s with only the unique identity index, +0.3s to recreate the rest). On big runs (same >=150k-ref gate as the resolver pool) the four non-unique edge indexes are now dropped for the batch loop and recreated in one pass each before synthesis. Why this is safe: - idx_edges_identity stays: INSERT OR IGNORE's dedup conflicts on it (#1034), and its leftmost column is `source`, so the only mid-window edge reads — resolution's supertype walks (implements/extends by source) — keep an index via its prefix (verified with EXPLAIN QUERY PLAN). - The window closes BEFORE synthesis, whose passes read kind-keyed, and on every error path (finally). - A crash inside the window heals on the next DatabaseConnection open — schema.sql re-applies CREATE INDEX IF NOT EXISTS, same recovery as the FTS bulk-load pattern this mirrors. - Concurrent readers (a daemon serving the project mid-index) stay correct; target/kind-keyed reads degrade to scans only for the window's duration. dubbo (4,402 files): persists 4.0s -> 3.0s, fresh init 11.9s -> ~11.1s, graph byte-identical. excalidraw (below the gate): untouched, byte-identical. Recreation cost ~250ms, logged under CODEGRAPH_SYNTH_TIMINGS as edge-index-recreate. Suite green (2444). Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
cf38ef65af
commit
567b4ad4be
@@ -151,6 +151,51 @@ export class DatabaseConnection {
|
||||
this.recreateFtsTriggers();
|
||||
}
|
||||
|
||||
/**
|
||||
* Names of the NON-UNIQUE edge indexes dropped for a bulk edge load.
|
||||
* idx_edges_identity deliberately stays: INSERT OR IGNORE's dedup conflicts
|
||||
* on it (#1034), and its leftmost column is `source`, so the source-keyed
|
||||
* reads resolution makes mid-window (supertype walks over
|
||||
* `implements`/`extends`) keep an index via its prefix — verified with
|
||||
* EXPLAIN QUERY PLAN. Target-keyed and kind-keyed reads (traversal,
|
||||
* synthesis) happen only after endBulkEdgeLoad().
|
||||
*/
|
||||
private static readonly BULK_EDGE_INDEX_NAMES = [
|
||||
'idx_edges_kind',
|
||||
'idx_edges_source_kind',
|
||||
'idx_edges_target_kind',
|
||||
'idx_edges_provenance',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Enter bulk-edge-load mode: drop the non-unique edge indexes so the mass
|
||||
* INSERT OR IGNORE stream pays one B-tree (the identity index) instead of
|
||||
* five — measured 2.8s → 1.1s inserting a 224k-edge resolution set, with
|
||||
* recreation costing ~0.3s. MUST be paired with endBulkEdgeLoad(); a crash
|
||||
* inside the window is healed on the next DatabaseConnection open (schema.sql
|
||||
* re-applies CREATE INDEX IF NOT EXISTS).
|
||||
*/
|
||||
beginBulkEdgeLoad(): void {
|
||||
for (const idx of DatabaseConnection.BULK_EDGE_INDEX_NAMES) {
|
||||
this.db.exec(`DROP INDEX IF EXISTS ${idx}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Leave bulk-edge-load mode: recreate the dropped indexes in one pass each
|
||||
* over the (now fully loaded) edges table — far cheaper than maintaining
|
||||
* them per-insert. DDL is extracted from schema.sql so it cannot drift.
|
||||
*/
|
||||
endBulkEdgeLoad(): void {
|
||||
const schemaPath = path.join(__dirname, 'schema.sql');
|
||||
const schema = fs.readFileSync(schemaPath, 'utf-8');
|
||||
for (const idx of DatabaseConnection.BULK_EDGE_INDEX_NAMES) {
|
||||
const m = schema.match(new RegExp(`CREATE INDEX IF NOT EXISTS ${idx}\\b[^;]*;`));
|
||||
if (!m) throw new Error(`schema.sql: edge index ${idx} not found for bulk-load recreation`);
|
||||
this.db.exec(m[0]);
|
||||
}
|
||||
}
|
||||
|
||||
/** Recreate the FTS triggers + rebuild if a bulk-load window never closed. */
|
||||
private healBulkNodeLoad(): void {
|
||||
const row = this.db
|
||||
|
||||
@@ -1149,6 +1149,15 @@ export class CodeGraph {
|
||||
): Promise<ResolutionResult> {
|
||||
return this.resolver.resolveAndPersistBatched(onProgress, undefined, onSynthesisProgress, {
|
||||
dbPath: this.db.getPath(),
|
||||
// Bulk-edge-load hooks: on big runs the resolver drops the non-unique
|
||||
// edge indexes for the batch loop and recreates them before synthesis
|
||||
// (which reads kind-keyed). Concurrent readers (a daemon serving this
|
||||
// project mid-index) stay CORRECT during the window — target/kind reads
|
||||
// just degrade to scans until the recreate.
|
||||
bulkEdgeLoad: {
|
||||
begin: () => this.db.beginBulkEdgeLoad(),
|
||||
end: () => this.db.endBulkEdgeLoad(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+30
-2
@@ -1336,8 +1336,10 @@ export class ReferenceResolver {
|
||||
// When provided, big batches fan out across a read-only resolver-worker
|
||||
// pool with results admitted in canonical order (see resolver-pool.ts).
|
||||
// Sequential fallback on any pool failure. CODEGRAPH_NO_PARALLEL_RESOLVE=1
|
||||
// disables entirely.
|
||||
parallel?: { dbPath: string }
|
||||
// disables entirely. bulkEdgeLoad hooks (when provided) bracket the batch
|
||||
// loop with drop/recreate of the non-unique edge indexes on big runs —
|
||||
// see DatabaseConnection.beginBulkEdgeLoad.
|
||||
parallel?: { dbPath: string; bulkEdgeLoad?: { begin: () => void; end: () => void } }
|
||||
): Promise<ResolutionResult> {
|
||||
// Resolution runs on the indexer's MAIN thread, and the #850 liveness
|
||||
// watchdog SIGKILLs a process whose event loop stalls past its window (60s
|
||||
@@ -1444,6 +1446,22 @@ export class ReferenceResolver {
|
||||
return this.resolveBatchYielding(batch, maybeYield);
|
||||
};
|
||||
|
||||
// Bulk edge load: on big runs, drop the non-unique edge indexes for the
|
||||
// duration of the batch loop (the identity index stays — OR IGNORE dedup
|
||||
// and the source-keyed supertype-walk reads both live on it). Recreated in
|
||||
// the inner finally BEFORE synthesis, whose passes read kind-keyed.
|
||||
// Measured on a 224k-edge resolution set: insert 2.8s → 1.1s + 0.3s
|
||||
// recreate. Same ref-count gate as the pool so small syncs never pay the
|
||||
// recreate cost.
|
||||
let bulkEdgesActive = false;
|
||||
if (parallel?.bulkEdgeLoad && total >= minRefsForPool()) {
|
||||
try {
|
||||
parallel.bulkEdgeLoad.begin();
|
||||
bulkEdgesActive = true;
|
||||
} catch { /* keep the indexes; inserts just pay the per-row maintenance */ }
|
||||
}
|
||||
|
||||
try {
|
||||
try {
|
||||
let batch = this.queries.getUnresolvedReferencesBatch(0, batchSize);
|
||||
let inFlight: InFlight | null = batch.length > 0 ? beginBatch(batch) : null;
|
||||
@@ -1559,6 +1577,16 @@ export class ReferenceResolver {
|
||||
batch = nextBatch;
|
||||
inFlight = nextInFlight;
|
||||
}
|
||||
} finally {
|
||||
// Recreate the edge indexes BEFORE synthesis (kind-keyed reads) and on
|
||||
// any error path. A crash before this line is healed by the next
|
||||
// DatabaseConnection open (schema.sql re-applies IF NOT EXISTS).
|
||||
if (bulkEdgesActive) {
|
||||
const tIdx = Date.now();
|
||||
parallel!.bulkEdgeLoad!.end();
|
||||
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] edge-index-recreate: ${Date.now() - tIdx}ms`);
|
||||
}
|
||||
}
|
||||
|
||||
// Dynamic-edge synthesis: now that all base `calls` edges are persisted,
|
||||
// synthesize observer/callback dispatch edges (dispatcher → registered
|
||||
|
||||
Reference in New Issue
Block a user