perf(resolution): batch-loop de-quadratic — keyset reads, changes-based guard, DB-scaled valve caps + resolve profiler (#1339)
The §7a.2 per-ref profile overturned the assumption the whole arc was built on: resolveOne owns only ~93s of the kernel-scale ~433s batch loop. Loop-stage attribution (CODEGRAPH_RESOLVE_PROFILE, shipped here) named the rest: backpressure folds 111.2s, count guard 93.9s, batch reads 54.6s, deletes/inserts/marks ~84s, settle 85.7s. - Non-progress guard O(remaining)→O(1): the per-batch COUNT(*) walked every remaining pending row (O(N²/batch) per run, 93.9s). The cleanup queries now return summed SQLite , and zero-removals-from-claimed-work is the guard signal — the DIRECT evidence the count diff inferred (a mismatched-name resolver makes keyed cleanup no-op ⇒ changes=0). A real COUNT runs only on that suspicious path and arbitrates exactly as before. - Batch reads OFFSET→keyset (54.6s→O(batch)): OFFSET re-walked the accumulated failed-row prefix every read; seeking past the last-seen rowid is prefix-independent and enumeration-order identical. - WAL valve caps scale with DB size (env still wins): every fold re-writes hot pages (#1231 in bounded form — 111.2s at the flat 256MB cap); soft=clamp(dbSize/4, 256MB, 2GB) trades ~4× fewer folds for a transient WAL ≈ project size. - CODEGRAPH_RESOLVE_PROFILE: per-outcome resolveOne histogram + loop-stage attribution, main + workers, off by default. Gates: dubbo dump byte-identical; suite 2,491 passed / 4 skipped (kernel required). Kernel-scale payoff run lands in the plan doc next. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
19cf1ec75b
commit
7cc23668b5
@@ -325,6 +325,17 @@ export class DatabaseConnection {
|
||||
}
|
||||
}
|
||||
|
||||
/** Size of the main DB file in bytes (0 for in-memory/unknown) — the WAL
|
||||
* valve scales its fold caps with it (resolveWalValveMb). */
|
||||
getDbFileSizeBytes(): number {
|
||||
if (!this.dbPath || this.dbPath === ':memory:') return 0;
|
||||
try {
|
||||
return fs.statSync(this.dbPath).size;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Current `wal_autocheckpoint` interval in pages (0 = disabled). */
|
||||
getWalAutocheckpoint(): number {
|
||||
const v = this.db.pragma('wal_autocheckpoint', { simple: true });
|
||||
|
||||
+55
-14
@@ -230,6 +230,7 @@ export class QueryBuilder {
|
||||
getNodesByLowerName?: SqliteStatement;
|
||||
getUnresolvedCount?: SqliteStatement;
|
||||
getUnresolvedBatch?: SqliteStatement;
|
||||
getUnresolvedBatchAfter?: SqliteStatement;
|
||||
deleteRefsByRowIdsFull?: SqliteStatement;
|
||||
getAllFilePaths?: SqliteStatement;
|
||||
getAllNodeNames?: SqliteStatement;
|
||||
@@ -2085,6 +2086,34 @@ export class QueryBuilder {
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Keyset variant of {@link getUnresolvedReferencesBatch} for the batched
|
||||
* resolution loop: seek past the last-seen row id instead of OFFSET-walking.
|
||||
* OFFSET reads re-scan the accumulated failed-row prefix on every batch —
|
||||
* O(failed rows) per read, measured at 54.6s of the kernel-scale batch loop
|
||||
* (§7a.2) — while the seek is O(batch) forever. `id` is the rowid alias, so
|
||||
* the enumeration order is identical to the OFFSET reader's.
|
||||
*/
|
||||
getUnresolvedReferencesBatchAfter(afterRowId: number, limit: number): UnresolvedReference[] {
|
||||
if (!this.stmts.getUnresolvedBatchAfter) {
|
||||
this.stmts.getUnresolvedBatchAfter = this.db.prepare(
|
||||
"SELECT * FROM unresolved_refs WHERE status = 'pending' AND id > ? ORDER BY id LIMIT ?"
|
||||
);
|
||||
}
|
||||
const rows = this.stmts.getUnresolvedBatchAfter.all(afterRowId, limit) as UnresolvedRefRow[];
|
||||
return rows.map((row) => ({
|
||||
fromNodeId: row.from_node_id,
|
||||
referenceName: row.reference_name,
|
||||
referenceKind: row.reference_kind as EdgeKind,
|
||||
line: row.line,
|
||||
column: row.col,
|
||||
candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined,
|
||||
filePath: row.file_path,
|
||||
language: row.language as Language,
|
||||
rowId: row.id,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all tracked file paths (lightweight — no full FileRecord objects)
|
||||
*/
|
||||
@@ -2182,17 +2211,22 @@ export class QueryBuilder {
|
||||
* Delete specific resolved references by (fromNodeId, referenceName, referenceKind) tuples.
|
||||
* More precise than deleteResolvedReferences — only removes refs that were actually resolved.
|
||||
*/
|
||||
deleteSpecificResolvedReferences(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): void {
|
||||
if (refs.length === 0) return;
|
||||
deleteSpecificResolvedReferences(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): number {
|
||||
if (refs.length === 0) return 0;
|
||||
const stmt = this.db.prepare(
|
||||
'DELETE FROM unresolved_refs WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?'
|
||||
);
|
||||
// Returns rows actually removed (SQLite `changes`, summed): the batched
|
||||
// resolution loop's non-progress guard keys on this — zero removals from
|
||||
// a batch that claimed work is the direct runaway signal (§7a.2).
|
||||
let changed = 0;
|
||||
const deleteMany = this.db.transaction((items: typeof refs) => {
|
||||
for (const ref of items) {
|
||||
stmt.run(ref.fromNodeId, ref.referenceName, ref.referenceKind);
|
||||
changed += stmt.run(ref.fromNodeId, ref.referenceName, ref.referenceKind).changes;
|
||||
}
|
||||
});
|
||||
deleteMany(refs);
|
||||
return changed;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2203,13 +2237,15 @@ export class QueryBuilder {
|
||||
* caller's same-named call sites, the later sites' edges were silently never
|
||||
* created (#1269).
|
||||
*/
|
||||
deleteReferencesByRowIds(rowIds: number[]): void {
|
||||
if (rowIds.length === 0) return;
|
||||
deleteReferencesByRowIds(rowIds: number[]): number {
|
||||
if (rowIds.length === 0) return 0;
|
||||
// One transaction for all chunks (each chunk was previously its own
|
||||
// implicit transaction = its own WAL commit — measurable on 100k+-ref
|
||||
// resolution persists), and the full-size chunk statement is cached so
|
||||
// repeat calls skip the re-prepare; only the final partial chunk (if any)
|
||||
// prepares ad hoc.
|
||||
// prepares ad hoc. Returns rows actually removed (summed `changes`) for
|
||||
// the batched loop's non-progress guard (§7a.2).
|
||||
let changed = 0;
|
||||
this.db.transaction(() => {
|
||||
for (let i = 0; i < rowIds.length; i += SQLITE_PARAM_CHUNK_SIZE) {
|
||||
const chunk = rowIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE);
|
||||
@@ -2220,13 +2256,14 @@ export class QueryBuilder {
|
||||
`DELETE FROM unresolved_refs WHERE id IN (${placeholders})`
|
||||
);
|
||||
}
|
||||
this.stmts.deleteRefsByRowIdsFull.run(...chunk);
|
||||
changed += this.stmts.deleteRefsByRowIdsFull.run(...chunk).changes;
|
||||
} else {
|
||||
const placeholders = chunk.map(() => '?').join(',');
|
||||
this.db.prepare(`DELETE FROM unresolved_refs WHERE id IN (${placeholders})`).run(...chunk);
|
||||
changed += this.db.prepare(`DELETE FROM unresolved_refs WHERE id IN (${placeholders})`).run(...chunk).changes;
|
||||
}
|
||||
}
|
||||
})();
|
||||
return changed;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2238,17 +2275,19 @@ export class QueryBuilder {
|
||||
* is (re)written here so rows inserted before the v8 migration get their
|
||||
* tail the first time they're attempted.
|
||||
*/
|
||||
markReferencesFailed(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): void {
|
||||
if (refs.length === 0) return;
|
||||
markReferencesFailed(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): number {
|
||||
if (refs.length === 0) return 0;
|
||||
const stmt = this.db.prepare(
|
||||
"UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?"
|
||||
);
|
||||
let changed = 0;
|
||||
const markMany = this.db.transaction((items: typeof refs) => {
|
||||
for (const ref of items) {
|
||||
stmt.run(referenceNameTail(ref.referenceName), ref.fromNodeId, ref.referenceName, ref.referenceKind);
|
||||
changed += stmt.run(referenceNameTail(ref.referenceName), ref.fromNodeId, ref.referenceName, ref.referenceKind).changes;
|
||||
}
|
||||
});
|
||||
markMany(refs);
|
||||
return changed;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2259,17 +2298,19 @@ export class QueryBuilder {
|
||||
* can differ per call site (receiver-type inference reads the ref's line),
|
||||
* so a sibling must not inherit this row's failure.
|
||||
*/
|
||||
markReferencesFailedByRowIds(refs: Array<{ rowId: number; referenceName: string }>): void {
|
||||
if (refs.length === 0) return;
|
||||
markReferencesFailedByRowIds(refs: Array<{ rowId: number; referenceName: string }>): number {
|
||||
if (refs.length === 0) return 0;
|
||||
const stmt = this.db.prepare(
|
||||
"UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE id = ?"
|
||||
);
|
||||
let changed = 0;
|
||||
const markMany = this.db.transaction((items: typeof refs) => {
|
||||
for (const ref of items) {
|
||||
stmt.run(referenceNameTail(ref.referenceName), ref.rowId);
|
||||
changed += stmt.run(referenceNameTail(ref.referenceName), ref.rowId).changes;
|
||||
}
|
||||
});
|
||||
markMany(refs);
|
||||
return changed;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+9
-1
@@ -61,11 +61,19 @@ const CHECK_INTERVAL_MS = 2000;
|
||||
* Resolve the valve's soft threshold from the `CODEGRAPH_WAL_VALVE_MB`
|
||||
* override; non-numeric / non-positive values fall back to the default.
|
||||
*/
|
||||
export function resolveWalValveMb(envVal: string | undefined): number {
|
||||
export function resolveWalValveMb(envVal: string | undefined, dbSizeBytes?: number): number {
|
||||
if (envVal !== undefined && envVal !== '') {
|
||||
const n = Number(envVal);
|
||||
if (Number.isFinite(n) && n > 0) return Math.floor(n);
|
||||
}
|
||||
// Scale with the project when the caller knows the DB size: every fold
|
||||
// re-writes hot B-tree pages into the main file (the #1231 pathology in
|
||||
// bounded form — 111s of a kernel-scale batch loop at the flat 256MB cap,
|
||||
// §7a.2), so a big project affords a proportionally bigger transient WAL
|
||||
// (~dbSize/4 soft ⇒ file cap ≈ dbSize) in exchange for ~4× fewer folds.
|
||||
if (dbSizeBytes !== undefined && dbSizeBytes > 0) {
|
||||
return Math.min(2048, Math.max(DEFAULT_WAL_VALVE_MB, Math.floor(dbSizeBytes / 4 / (1024 * 1024))));
|
||||
}
|
||||
return DEFAULT_WAL_VALVE_MB;
|
||||
}
|
||||
|
||||
|
||||
+3
-3
@@ -25,7 +25,7 @@ import {
|
||||
FindRelevantContextOptions,
|
||||
} from './types';
|
||||
import { DatabaseConnection, getDatabasePath, removeDatabaseFiles } from './db';
|
||||
import { WalCheckpointValve } from './db/wal-valve';
|
||||
import { WalCheckpointValve, resolveWalValveMb } from './db/wal-valve';
|
||||
import { QueryBuilder } from './db/queries';
|
||||
import {
|
||||
isInitialized,
|
||||
@@ -472,7 +472,7 @@ export class CodeGraph {
|
||||
this.db.setWalAutocheckpoint(0);
|
||||
walValve = new WalCheckpointValve(
|
||||
this.db,
|
||||
undefined,
|
||||
resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB, this.db.getDbFileSizeBytes()),
|
||||
undefined,
|
||||
options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
|
||||
);
|
||||
@@ -753,7 +753,7 @@ export class CodeGraph {
|
||||
this.db.setWalAutocheckpoint(0);
|
||||
walValve = new WalCheckpointValve(
|
||||
this.db,
|
||||
undefined,
|
||||
resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB, this.db.getDbFileSizeBytes()),
|
||||
undefined,
|
||||
options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
|
||||
);
|
||||
|
||||
+101
-14
@@ -637,7 +637,7 @@ export class ReferenceResolver {
|
||||
|
||||
for (let i = 0; i < refs.length; i++) {
|
||||
const ref = refs[i]!; // Array index is guaranteed to be in bounds
|
||||
const result = this.resolveOne(ref);
|
||||
const result = this.resolveOneTimed(ref);
|
||||
|
||||
if (result) {
|
||||
resolved.push(result);
|
||||
@@ -1233,7 +1233,7 @@ export class ReferenceResolver {
|
||||
language: raw.language || this.getLanguageFromNodeId(raw.fromNodeId),
|
||||
rowId: raw.rowId,
|
||||
};
|
||||
const result = this.resolveOne(ref);
|
||||
const result = this.resolveOneTimed(ref);
|
||||
if (result) {
|
||||
resolved.push(result);
|
||||
byMethod[result.resolvedBy] = (byMethod[result.resolvedBy] || 0) + 1;
|
||||
@@ -1266,6 +1266,47 @@ export class ReferenceResolver {
|
||||
* of resolveBatchYielding, minus the main-thread yields (worker threads have
|
||||
* no watchdog heartbeat to starve). Results are in input order.
|
||||
*/
|
||||
/**
|
||||
* CODEGRAPH_RESOLVE_PROFILE=1: per-outcome wall-clock histogram of
|
||||
* resolveOne, keyed by the winning strategy (`resolvedBy`) or
|
||||
* `fail:<referenceKind>` — the §7a.2 "profile the per-ref path" probe. The
|
||||
* kernel-scale batch loop is ~430s and CORE-INVARIANT (835.9s pooled-4-on-8
|
||||
* ≈ 812.5s sequential-on-2 for the whole superphase), so the next lever is
|
||||
* which CLASS of ref the time belongs to, not more parallelism. Off by
|
||||
* default: the hrtime pair costs ~100ns/ref only when the env is set.
|
||||
*/
|
||||
private resolveProfile: Map<string, { n: number; ns: bigint }> | null =
|
||||
process.env.CODEGRAPH_RESOLVE_PROFILE ? new Map() : null;
|
||||
|
||||
private resolveOneTimed(ref: UnresolvedRef): ResolvedRef | null {
|
||||
if (!this.resolveProfile) return this.resolveOne(ref);
|
||||
const t0 = process.hrtime.bigint();
|
||||
const result = this.resolveOne(ref);
|
||||
const dt = process.hrtime.bigint() - t0;
|
||||
const key = result ? result.resolvedBy : `fail:${ref.referenceKind}`;
|
||||
const slot = this.resolveProfile.get(key);
|
||||
if (slot) {
|
||||
slot.n++;
|
||||
slot.ns += dt;
|
||||
} else {
|
||||
this.resolveProfile.set(key, { n: 1, ns: dt });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Dump the CODEGRAPH_RESOLVE_PROFILE histogram to stderr (no-op when off). */
|
||||
dumpResolveProfile(label: string): void {
|
||||
if (!this.resolveProfile || this.resolveProfile.size === 0) return;
|
||||
const rows = [...this.resolveProfile.entries()]
|
||||
.map(([k, v]) => ({ k, n: v.n, ms: Number(v.ns / 1_000_000n) }))
|
||||
.sort((a, b) => b.ms - a.ms);
|
||||
for (const r of rows) {
|
||||
console.error(
|
||||
`[resolve-profile] ${label} ${r.k}: n=${r.n} total=${(r.ms / 1000).toFixed(1)}s avg=${((r.ms * 1000) / Math.max(1, r.n)).toFixed(0)}µs`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
resolveListForAdmission(refs: UnresolvedReference[]): {
|
||||
resolved: ResolvedRef[];
|
||||
unresolved: UnresolvedRef[];
|
||||
@@ -1288,7 +1329,7 @@ export class ReferenceResolver {
|
||||
language: raw.language || this.getLanguageFromNodeId(raw.fromNodeId),
|
||||
rowId: raw.rowId,
|
||||
};
|
||||
const result = this.resolveOne(ref);
|
||||
const result = this.resolveOneTimed(ref);
|
||||
if (result) {
|
||||
resolved.push(result);
|
||||
byMethod[result.resolvedBy] = (byMethod[result.resolvedBy] || 0) + 1;
|
||||
@@ -1362,6 +1403,16 @@ export class ReferenceResolver {
|
||||
console.error(`[pool-timing] backpressure hook: ${parallel?.backpressure ? 'present' : 'absent'}`);
|
||||
}
|
||||
|
||||
// CODEGRAPH_RESOLVE_PROFILE loop-stage attribution: the §7a.2 kernel-scale
|
||||
// histogram showed resolveOne owns only ~93s of the ~436s batch loop —
|
||||
// these counters name where the other ~340s goes (reads, edge build+insert,
|
||||
// deletes/marks, the per-batch count guard).
|
||||
const loopProf: Record<string, number> | null = process.env.CODEGRAPH_RESOLVE_PROFILE
|
||||
? { read: 0, settle: 0, backpressure: 0, createEdges: 0, insertEdges: 0, deletes: 0, marks: 0, countGuard: 0 }
|
||||
: null;
|
||||
const lp = (k: string, t0: number): void => { if (loopProf) loopProf[k] = (loopProf[k] ?? 0) + (Date.now() - t0); };
|
||||
let tLp = 0;
|
||||
|
||||
await this.warmCachesYielding(maybeYield);
|
||||
|
||||
const total = this.queries.getUnresolvedReferencesCount();
|
||||
@@ -1476,18 +1527,24 @@ export class ReferenceResolver {
|
||||
|
||||
try {
|
||||
try {
|
||||
let batch = this.queries.getUnresolvedReferencesBatch(0, batchSize);
|
||||
tLp = Date.now();
|
||||
let batch = this.queries.getUnresolvedReferencesBatchAfter(0, batchSize);
|
||||
lp('read', tLp);
|
||||
let inFlight: InFlight | null = batch.length > 0 ? beginBatch(batch) : null;
|
||||
while (batch.length > 0 && inFlight) {
|
||||
// Prefetch the NEXT batch before this one persists: this batch's rows
|
||||
// are still pending (nothing has mutated the table since they were
|
||||
// read), so skipping exactly batch.length rows in the same rowid
|
||||
// enumeration yields the following batch.
|
||||
const nextBatch = this.queries.getUnresolvedReferencesBatch(batch.length, batchSize);
|
||||
// read), so seeking past this batch's last row id in the same rowid
|
||||
// enumeration yields the following batch (keyset — OFFSET re-walked the
|
||||
// accumulated failed prefix every read, 54.6s at kernel scale, §7a.2).
|
||||
tLp = Date.now();
|
||||
const nextBatch = this.queries.getUnresolvedReferencesBatchAfter(batch[batch.length - 1]!.rowId!, batchSize);
|
||||
lp('read', tLp);
|
||||
|
||||
const tBatch = Date.now();
|
||||
const result = await settleBatch(inFlight, batch);
|
||||
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[pool-timing] batch ${inFlight.mode}: ${batch.length} refs in ${Date.now() - tBatch}ms`);
|
||||
lp('settle', tBatch);
|
||||
|
||||
// WAL-valve backstop at the ONE pool-idle boundary of the double-buffer
|
||||
// (this batch settled, the next not yet fanned out): past the hard cap
|
||||
@@ -1495,8 +1552,10 @@ export class ReferenceResolver {
|
||||
// are all between statements — so the backfill completes, readers
|
||||
// re-enter at SQLite's backfilled mark, and the next persist commit
|
||||
// WRAPS the WAL instead of growing it. No-op (one fstat) under the cap.
|
||||
tLp = Date.now();
|
||||
const bp = parallel?.backpressure?.();
|
||||
if (bp) await bp;
|
||||
lp('backpressure', tLp);
|
||||
|
||||
// Persist in bounded sub-transactions with yields between: a whole
|
||||
// batch's edge insert / keyed deletes are otherwise one solid
|
||||
@@ -1515,11 +1574,15 @@ export class ReferenceResolver {
|
||||
// base class if those edges are visible. (Validated on dubbo: fanning
|
||||
// out first downgraded exactly those supertype-method resolutions from
|
||||
// the 0.9 typed-receiver path to the 0.65 word-overlap fallback.)
|
||||
tLp = Date.now();
|
||||
const edges = this.createEdges(result.resolved);
|
||||
lp('createEdges', tLp);
|
||||
tLp = Date.now();
|
||||
for (let i = 0; i < edges.length; i += PERSIST_CHUNK) {
|
||||
this.queries.insertEdges(edges.slice(i, i + PERSIST_CHUNK));
|
||||
await maybeYield();
|
||||
}
|
||||
lp('insertEdges', tLp);
|
||||
|
||||
// NOW fan the next batch out — workers see exactly the edge state the
|
||||
// sequential baseline would (every batch ≤ this one committed), while
|
||||
@@ -1531,29 +1594,34 @@ export class ReferenceResolver {
|
||||
// by row id, so a same-key sibling ref in a LATER batch (same caller
|
||||
// calling the same callee at another line) is left pending for its own
|
||||
// attempt instead of being swept out with this batch's rows (#1269).
|
||||
tLp = Date.now();
|
||||
let removedThisBatch = 0;
|
||||
const resolvedCleanup = ReferenceResolver.partitionResolvedCleanup(result.resolved);
|
||||
for (let i = 0; i < resolvedCleanup.rowIds.length; i += PERSIST_CHUNK) {
|
||||
this.queries.deleteReferencesByRowIds(resolvedCleanup.rowIds.slice(i, i + PERSIST_CHUNK));
|
||||
removedThisBatch += this.queries.deleteReferencesByRowIds(resolvedCleanup.rowIds.slice(i, i + PERSIST_CHUNK));
|
||||
await maybeYield();
|
||||
}
|
||||
for (let i = 0; i < resolvedCleanup.legacyKeys.length; i += PERSIST_CHUNK) {
|
||||
this.queries.deleteSpecificResolvedReferences(resolvedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
||||
removedThisBatch += this.queries.deleteSpecificResolvedReferences(resolvedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
||||
await maybeYield();
|
||||
}
|
||||
lp('deletes', tLp);
|
||||
|
||||
// Park unresolvable refs from this batch as status='failed' so they
|
||||
// leave the pending set (the batch reader and non-progress guard below
|
||||
// only see pending rows) but stay retryable when a later sync adds a
|
||||
// symbol that could satisfy them (#1240).
|
||||
tLp = Date.now();
|
||||
const failedCleanup = ReferenceResolver.partitionFailedCleanup(result.unresolved);
|
||||
for (let i = 0; i < failedCleanup.byRowId.length; i += PERSIST_CHUNK) {
|
||||
this.queries.markReferencesFailedByRowIds(failedCleanup.byRowId.slice(i, i + PERSIST_CHUNK));
|
||||
removedThisBatch += this.queries.markReferencesFailedByRowIds(failedCleanup.byRowId.slice(i, i + PERSIST_CHUNK));
|
||||
await maybeYield();
|
||||
}
|
||||
for (let i = 0; i < failedCleanup.legacyKeys.length; i += PERSIST_CHUNK) {
|
||||
this.queries.markReferencesFailed(failedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
||||
removedThisBatch += this.queries.markReferencesFailed(failedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
||||
await maybeYield();
|
||||
}
|
||||
lp('marks', tLp);
|
||||
|
||||
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[pool-timing] batch persist: ${Date.now() - tPersist}ms`);
|
||||
|
||||
@@ -1590,9 +1658,22 @@ export class ReferenceResolver {
|
||||
// 1.4 GB before the Go-fallback fix). Stop rather than grow the graph
|
||||
// without bound. (An in-flight prefetched batch is abandoned unsettled —
|
||||
// fan-out has no side effects until settleBatch appends its results.)
|
||||
const remaining = this.queries.getUnresolvedReferencesCount();
|
||||
if (remaining >= prevRemaining) break;
|
||||
prevRemaining = remaining;
|
||||
// Non-progress signal, now O(1): `changes` summed across this batch's
|
||||
// deletes + failed-parks is the DIRECT evidence the guard's old count
|
||||
// diff inferred — a resolver returning a mismatched name makes the keyed
|
||||
// cleanup no-op, which shows up here as zero removals. The per-batch
|
||||
// COUNT(*) it replaces walked every remaining pending row — O(N²/batch)
|
||||
// over a run, 93.9s of the kernel-scale batch loop (§7a.2). A REAL count
|
||||
// runs only on the suspicious path (claimed-work batch removed nothing —
|
||||
// e.g. every row was a sibling a legacy-key sweep already consumed),
|
||||
// where it arbitrates stop-vs-continue exactly as before.
|
||||
if (removedThisBatch <= 0 && batch.length > 0) {
|
||||
tLp = Date.now();
|
||||
const remaining = this.queries.getUnresolvedReferencesCount();
|
||||
lp('countGuard', tLp);
|
||||
if (remaining >= prevRemaining) break;
|
||||
prevRemaining = remaining;
|
||||
}
|
||||
|
||||
// Advance the pipeline: the prefetched batch (already fanned out when
|
||||
// the pool is on) becomes the current one.
|
||||
@@ -1639,6 +1720,12 @@ export class ReferenceResolver {
|
||||
if (pool) await pool.destroy().catch(() => undefined);
|
||||
}
|
||||
|
||||
if (loopProf) {
|
||||
const parts = Object.entries(loopProf).map(([k, v]) => `${k}=${(v / 1000).toFixed(1)}s`).join(' ');
|
||||
console.error(`[resolve-profile] loop-stages ${parts}`);
|
||||
}
|
||||
this.dumpResolveProfile('main');
|
||||
|
||||
return {
|
||||
resolved: [],
|
||||
unresolved: [],
|
||||
|
||||
@@ -20,7 +20,7 @@ try {
|
||||
(require('node:module') as { enableCompileCache?: () => void }).enableCompileCache?.();
|
||||
} catch { /* cache is best-effort */ }
|
||||
|
||||
import { parentPort } from 'worker_threads';
|
||||
import { parentPort, threadId } from 'worker_threads';
|
||||
import { createDatabase, SqliteDatabase } from '../db/sqlite-adapter';
|
||||
import { QueryBuilder } from '../db/queries';
|
||||
import { ReferenceResolver } from './index';
|
||||
@@ -95,6 +95,9 @@ port.on('message', (msg: InMessage) => {
|
||||
break;
|
||||
}
|
||||
case 'close': {
|
||||
try {
|
||||
resolver?.dumpResolveProfile(`worker#${threadId}`);
|
||||
} catch { /* diagnostics never block shutdown */ }
|
||||
try {
|
||||
db?.close();
|
||||
} catch {
|
||||
|
||||
Reference in New Issue
Block a user