perf(synthesis): fan dynamic-dispatch passes across the resolver pool, byte-identical graphs (#1321)
The ~36 independent synthesis passes (callback/event/framework wiring) ran sequentially on the indexer's main thread — 2.0s of a 4,402-file Java repo's index, and the stage where kernel-class repos die (#1212). They now live in an explicit registry (SYNTH_PASSES) and, when the resolver pool is alive (>=150k-ref repos), fan out across its read-only workers: dubbo synthesis 2,024ms -> ~900ms (-55%), total fresh init 13.5s -> 11.9s. Graphs verified byte-for-byte identical on both the pool path (dubbo) and the sequential path (excalidraw). Why this is safe: no pass's edges persist until the ordered merge, so every pass sees the same committed post-resolution DB state in either mode, and results merge in registry order regardless of completion order — the first-seen dedup is unchanged. The pool now survives through synthesis (destroy moved after it) instead of being torn down moments before the one stage that could reuse it. Robustness: a pass that fails on a worker (crash, OOM) is retried on the main thread — a synthesizer blow-up now costs one worker instead of the whole index, which is half the #1212 story on very large repos. Also: ref-row cleanup deletes now run as one transaction with a cached statement instead of one implicit commit per 500-row chunk (mechanically fewer WAL commits; matters most on HDD-class storage). A set-based rewrite of failed-ref parking was tried, measured ~zero on NVMe, and dropped — the remaining persist cost is edge-index B-tree maintenance, not statement dispatch. SYNTH_PROGRESS_STEPS now derives from the registry (passes + fixed marks); the pin test counts registry entries plus literal __mark sites. Suite green (2444). Sequential-path timing unchanged on excalidraw. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
a2f3c31a97
commit
cf38ef65af
@@ -24,6 +24,8 @@ import { parentPort } from 'worker_threads';
|
||||
import { createDatabase, SqliteDatabase } from '../db/sqlite-adapter';
|
||||
import { QueryBuilder } from '../db/queries';
|
||||
import { ReferenceResolver } from './index';
|
||||
import { SYNTH_PASSES } from './callback-synthesizer';
|
||||
import { createYielder } from './cooperative-yield';
|
||||
import type { UnresolvedReference } from '../types';
|
||||
|
||||
if (!parentPort) {
|
||||
@@ -32,11 +34,13 @@ if (!parentPort) {
|
||||
const port = parentPort;
|
||||
|
||||
let db: SqliteDatabase | null = null;
|
||||
let queries: QueryBuilder | null = null;
|
||||
let resolver: ReferenceResolver | null = null;
|
||||
|
||||
type InMessage =
|
||||
| { type: 'open'; dbPath: string; projectRoot: string }
|
||||
| { type: 'resolve'; id: number; refs: UnresolvedReference[] }
|
||||
| { type: 'synth'; id: number; pass: string }
|
||||
| { type: 'close' };
|
||||
|
||||
port.on('message', (msg: InMessage) => {
|
||||
@@ -49,7 +53,7 @@ port.on('message', (msg: InMessage) => {
|
||||
db.pragma('busy_timeout = 5000');
|
||||
db.pragma('cache_size = -32000');
|
||||
const tDb = Date.now();
|
||||
const queries = new QueryBuilder(db);
|
||||
queries = new QueryBuilder(db);
|
||||
resolver = new ReferenceResolver(msg.projectRoot, queries);
|
||||
resolver.initialize();
|
||||
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[pool-timing] worker open: db=${tDb - tOpen}ms init=${Date.now() - tDb}ms`);
|
||||
@@ -64,6 +68,32 @@ port.on('message', (msg: InMessage) => {
|
||||
port.postMessage({ type: 'result', id: msg.id, ...out });
|
||||
break;
|
||||
}
|
||||
case 'synth': {
|
||||
// Run one synthesis pass against this worker's read-only connection.
|
||||
// Passes only READ (graph + source via the resolver's context); their
|
||||
// edges are returned for the main thread's ordered merge. Async, with
|
||||
// its own error propagation — a throwing pass reports {type:'error'}
|
||||
// and the main thread retries it sequentially.
|
||||
if (!resolver || !queries) throw new Error('resolver-worker: synth before open');
|
||||
const pass = SYNTH_PASSES.find((p) => p.name === msg.pass);
|
||||
if (!pass) throw new Error(`resolver-worker: unknown synth pass '${msg.pass}'`);
|
||||
const q = queries;
|
||||
const r = resolver;
|
||||
void (async () => {
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const edges = await pass.run(q, r.getResolutionContext(), createYielder());
|
||||
port.postMessage({ type: 'synth-result', id: msg.id, edges, ms: Date.now() - t0 });
|
||||
} catch (err) {
|
||||
port.postMessage({
|
||||
type: 'error',
|
||||
id: msg.id,
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
})();
|
||||
break;
|
||||
}
|
||||
case 'close': {
|
||||
try {
|
||||
db?.close();
|
||||
|
||||
Reference in New Issue
Block a user