perf(resolution): batch-loop de-quadratic — keyset reads, changes-based guard, DB-scaled valve caps + resolve profiler (#1339)
The §7a.2 per-ref profile overturned the assumption the whole arc was built on: resolveOne owns only ~93s of the kernel-scale ~433s batch loop. Loop-stage attribution (CODEGRAPH_RESOLVE_PROFILE, shipped here) named the rest: backpressure folds 111.2s, count guard 93.9s, batch reads 54.6s, deletes/inserts/marks ~84s, settle 85.7s. - Non-progress guard O(remaining)→O(1): the per-batch COUNT(*) walked every remaining pending row (O(N²/batch) per run, 93.9s). The cleanup queries now return summed SQLite , and zero-removals-from-claimed-work is the guard signal — the DIRECT evidence the count diff inferred (a mismatched-name resolver makes keyed cleanup no-op ⇒ changes=0). A real COUNT runs only on that suspicious path and arbitrates exactly as before. - Batch reads OFFSET→keyset (54.6s→O(batch)): OFFSET re-walked the accumulated failed-row prefix every read; seeking past the last-seen rowid is prefix-independent and enumeration-order identical. - WAL valve caps scale with DB size (env still wins): every fold re-writes hot pages (#1231 in bounded form — 111.2s at the flat 256MB cap); soft=clamp(dbSize/4, 256MB, 2GB) trades ~4× fewer folds for a transient WAL ≈ project size. - CODEGRAPH_RESOLVE_PROFILE: per-outcome resolveOne histogram + loop-stage attribution, main + workers, off by default. Gates: dubbo dump byte-identical; suite 2,491 passed / 4 skipped (kernel required). Kernel-scale payoff run lands in the plan doc next. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
19cf1ec75b
commit
7cc23668b5
@@ -416,3 +416,14 @@ describe('valve file-size trigger (§7a.1: backfilled WAL still grows the file)'
|
|||||||
db.close();
|
db.close();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('resolveWalValveMb DB-size scaling (§7a.2 fold-tax reduction)', () => {
|
||||||
|
it('scales soft cap ~dbSize/4 within [256, 2048]MB; env always wins', () => {
|
||||||
|
const GB = 1024 * 1024 * 1024;
|
||||||
|
expect(resolveWalValveMb(undefined, 100 * 1024 * 1024)).toBe(256); // floor
|
||||||
|
expect(resolveWalValveMb(undefined, 4.6 * GB)).toBe(1177); // ~dbSize/4
|
||||||
|
expect(resolveWalValveMb(undefined, 40 * GB)).toBe(2048); // ceiling
|
||||||
|
expect(resolveWalValveMb('64', 40 * GB)).toBe(64); // env override wins
|
||||||
|
expect(resolveWalValveMb(undefined, 0)).toBe(256); // unknown size → default
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -325,6 +325,17 @@ export class DatabaseConnection {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Size of the main DB file in bytes (0 for in-memory/unknown) — the WAL
|
||||||
|
* valve scales its fold caps with it (resolveWalValveMb). */
|
||||||
|
getDbFileSizeBytes(): number {
|
||||||
|
if (!this.dbPath || this.dbPath === ':memory:') return 0;
|
||||||
|
try {
|
||||||
|
return fs.statSync(this.dbPath).size;
|
||||||
|
} catch {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Current `wal_autocheckpoint` interval in pages (0 = disabled). */
|
/** Current `wal_autocheckpoint` interval in pages (0 = disabled). */
|
||||||
getWalAutocheckpoint(): number {
|
getWalAutocheckpoint(): number {
|
||||||
const v = this.db.pragma('wal_autocheckpoint', { simple: true });
|
const v = this.db.pragma('wal_autocheckpoint', { simple: true });
|
||||||
|
|||||||
+55
-14
@@ -230,6 +230,7 @@ export class QueryBuilder {
|
|||||||
getNodesByLowerName?: SqliteStatement;
|
getNodesByLowerName?: SqliteStatement;
|
||||||
getUnresolvedCount?: SqliteStatement;
|
getUnresolvedCount?: SqliteStatement;
|
||||||
getUnresolvedBatch?: SqliteStatement;
|
getUnresolvedBatch?: SqliteStatement;
|
||||||
|
getUnresolvedBatchAfter?: SqliteStatement;
|
||||||
deleteRefsByRowIdsFull?: SqliteStatement;
|
deleteRefsByRowIdsFull?: SqliteStatement;
|
||||||
getAllFilePaths?: SqliteStatement;
|
getAllFilePaths?: SqliteStatement;
|
||||||
getAllNodeNames?: SqliteStatement;
|
getAllNodeNames?: SqliteStatement;
|
||||||
@@ -2085,6 +2086,34 @@ export class QueryBuilder {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keyset variant of {@link getUnresolvedReferencesBatch} for the batched
|
||||||
|
* resolution loop: seek past the last-seen row id instead of OFFSET-walking.
|
||||||
|
* OFFSET reads re-scan the accumulated failed-row prefix on every batch —
|
||||||
|
* O(failed rows) per read, measured at 54.6s of the kernel-scale batch loop
|
||||||
|
* (§7a.2) — while the seek is O(batch) forever. `id` is the rowid alias, so
|
||||||
|
* the enumeration order is identical to the OFFSET reader's.
|
||||||
|
*/
|
||||||
|
getUnresolvedReferencesBatchAfter(afterRowId: number, limit: number): UnresolvedReference[] {
|
||||||
|
if (!this.stmts.getUnresolvedBatchAfter) {
|
||||||
|
this.stmts.getUnresolvedBatchAfter = this.db.prepare(
|
||||||
|
"SELECT * FROM unresolved_refs WHERE status = 'pending' AND id > ? ORDER BY id LIMIT ?"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const rows = this.stmts.getUnresolvedBatchAfter.all(afterRowId, limit) as UnresolvedRefRow[];
|
||||||
|
return rows.map((row) => ({
|
||||||
|
fromNodeId: row.from_node_id,
|
||||||
|
referenceName: row.reference_name,
|
||||||
|
referenceKind: row.reference_kind as EdgeKind,
|
||||||
|
line: row.line,
|
||||||
|
column: row.col,
|
||||||
|
candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined,
|
||||||
|
filePath: row.file_path,
|
||||||
|
language: row.language as Language,
|
||||||
|
rowId: row.id,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get all tracked file paths (lightweight — no full FileRecord objects)
|
* Get all tracked file paths (lightweight — no full FileRecord objects)
|
||||||
*/
|
*/
|
||||||
@@ -2182,17 +2211,22 @@ export class QueryBuilder {
|
|||||||
* Delete specific resolved references by (fromNodeId, referenceName, referenceKind) tuples.
|
* Delete specific resolved references by (fromNodeId, referenceName, referenceKind) tuples.
|
||||||
* More precise than deleteResolvedReferences — only removes refs that were actually resolved.
|
* More precise than deleteResolvedReferences — only removes refs that were actually resolved.
|
||||||
*/
|
*/
|
||||||
deleteSpecificResolvedReferences(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): void {
|
deleteSpecificResolvedReferences(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): number {
|
||||||
if (refs.length === 0) return;
|
if (refs.length === 0) return 0;
|
||||||
const stmt = this.db.prepare(
|
const stmt = this.db.prepare(
|
||||||
'DELETE FROM unresolved_refs WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?'
|
'DELETE FROM unresolved_refs WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?'
|
||||||
);
|
);
|
||||||
|
// Returns rows actually removed (SQLite `changes`, summed): the batched
|
||||||
|
// resolution loop's non-progress guard keys on this — zero removals from
|
||||||
|
// a batch that claimed work is the direct runaway signal (§7a.2).
|
||||||
|
let changed = 0;
|
||||||
const deleteMany = this.db.transaction((items: typeof refs) => {
|
const deleteMany = this.db.transaction((items: typeof refs) => {
|
||||||
for (const ref of items) {
|
for (const ref of items) {
|
||||||
stmt.run(ref.fromNodeId, ref.referenceName, ref.referenceKind);
|
changed += stmt.run(ref.fromNodeId, ref.referenceName, ref.referenceKind).changes;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
deleteMany(refs);
|
deleteMany(refs);
|
||||||
|
return changed;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -2203,13 +2237,15 @@ export class QueryBuilder {
|
|||||||
* caller's same-named call sites, the later sites' edges were silently never
|
* caller's same-named call sites, the later sites' edges were silently never
|
||||||
* created (#1269).
|
* created (#1269).
|
||||||
*/
|
*/
|
||||||
deleteReferencesByRowIds(rowIds: number[]): void {
|
deleteReferencesByRowIds(rowIds: number[]): number {
|
||||||
if (rowIds.length === 0) return;
|
if (rowIds.length === 0) return 0;
|
||||||
// One transaction for all chunks (each chunk was previously its own
|
// One transaction for all chunks (each chunk was previously its own
|
||||||
// implicit transaction = its own WAL commit — measurable on 100k+-ref
|
// implicit transaction = its own WAL commit — measurable on 100k+-ref
|
||||||
// resolution persists), and the full-size chunk statement is cached so
|
// resolution persists), and the full-size chunk statement is cached so
|
||||||
// repeat calls skip the re-prepare; only the final partial chunk (if any)
|
// repeat calls skip the re-prepare; only the final partial chunk (if any)
|
||||||
// prepares ad hoc.
|
// prepares ad hoc. Returns rows actually removed (summed `changes`) for
|
||||||
|
// the batched loop's non-progress guard (§7a.2).
|
||||||
|
let changed = 0;
|
||||||
this.db.transaction(() => {
|
this.db.transaction(() => {
|
||||||
for (let i = 0; i < rowIds.length; i += SQLITE_PARAM_CHUNK_SIZE) {
|
for (let i = 0; i < rowIds.length; i += SQLITE_PARAM_CHUNK_SIZE) {
|
||||||
const chunk = rowIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE);
|
const chunk = rowIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE);
|
||||||
@@ -2220,13 +2256,14 @@ export class QueryBuilder {
|
|||||||
`DELETE FROM unresolved_refs WHERE id IN (${placeholders})`
|
`DELETE FROM unresolved_refs WHERE id IN (${placeholders})`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
this.stmts.deleteRefsByRowIdsFull.run(...chunk);
|
changed += this.stmts.deleteRefsByRowIdsFull.run(...chunk).changes;
|
||||||
} else {
|
} else {
|
||||||
const placeholders = chunk.map(() => '?').join(',');
|
const placeholders = chunk.map(() => '?').join(',');
|
||||||
this.db.prepare(`DELETE FROM unresolved_refs WHERE id IN (${placeholders})`).run(...chunk);
|
changed += this.db.prepare(`DELETE FROM unresolved_refs WHERE id IN (${placeholders})`).run(...chunk).changes;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
|
return changed;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -2238,17 +2275,19 @@ export class QueryBuilder {
|
|||||||
* is (re)written here so rows inserted before the v8 migration get their
|
* is (re)written here so rows inserted before the v8 migration get their
|
||||||
* tail the first time they're attempted.
|
* tail the first time they're attempted.
|
||||||
*/
|
*/
|
||||||
markReferencesFailed(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): void {
|
markReferencesFailed(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): number {
|
||||||
if (refs.length === 0) return;
|
if (refs.length === 0) return 0;
|
||||||
const stmt = this.db.prepare(
|
const stmt = this.db.prepare(
|
||||||
"UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?"
|
"UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?"
|
||||||
);
|
);
|
||||||
|
let changed = 0;
|
||||||
const markMany = this.db.transaction((items: typeof refs) => {
|
const markMany = this.db.transaction((items: typeof refs) => {
|
||||||
for (const ref of items) {
|
for (const ref of items) {
|
||||||
stmt.run(referenceNameTail(ref.referenceName), ref.fromNodeId, ref.referenceName, ref.referenceKind);
|
changed += stmt.run(referenceNameTail(ref.referenceName), ref.fromNodeId, ref.referenceName, ref.referenceKind).changes;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
markMany(refs);
|
markMany(refs);
|
||||||
|
return changed;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -2259,17 +2298,19 @@ export class QueryBuilder {
|
|||||||
* can differ per call site (receiver-type inference reads the ref's line),
|
* can differ per call site (receiver-type inference reads the ref's line),
|
||||||
* so a sibling must not inherit this row's failure.
|
* so a sibling must not inherit this row's failure.
|
||||||
*/
|
*/
|
||||||
markReferencesFailedByRowIds(refs: Array<{ rowId: number; referenceName: string }>): void {
|
markReferencesFailedByRowIds(refs: Array<{ rowId: number; referenceName: string }>): number {
|
||||||
if (refs.length === 0) return;
|
if (refs.length === 0) return 0;
|
||||||
const stmt = this.db.prepare(
|
const stmt = this.db.prepare(
|
||||||
"UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE id = ?"
|
"UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE id = ?"
|
||||||
);
|
);
|
||||||
|
let changed = 0;
|
||||||
const markMany = this.db.transaction((items: typeof refs) => {
|
const markMany = this.db.transaction((items: typeof refs) => {
|
||||||
for (const ref of items) {
|
for (const ref of items) {
|
||||||
stmt.run(referenceNameTail(ref.referenceName), ref.rowId);
|
changed += stmt.run(referenceNameTail(ref.referenceName), ref.rowId).changes;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
markMany(refs);
|
markMany(refs);
|
||||||
|
return changed;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
+9
-1
@@ -61,11 +61,19 @@ const CHECK_INTERVAL_MS = 2000;
|
|||||||
* Resolve the valve's soft threshold from the `CODEGRAPH_WAL_VALVE_MB`
|
* Resolve the valve's soft threshold from the `CODEGRAPH_WAL_VALVE_MB`
|
||||||
* override; non-numeric / non-positive values fall back to the default.
|
* override; non-numeric / non-positive values fall back to the default.
|
||||||
*/
|
*/
|
||||||
export function resolveWalValveMb(envVal: string | undefined): number {
|
export function resolveWalValveMb(envVal: string | undefined, dbSizeBytes?: number): number {
|
||||||
if (envVal !== undefined && envVal !== '') {
|
if (envVal !== undefined && envVal !== '') {
|
||||||
const n = Number(envVal);
|
const n = Number(envVal);
|
||||||
if (Number.isFinite(n) && n > 0) return Math.floor(n);
|
if (Number.isFinite(n) && n > 0) return Math.floor(n);
|
||||||
}
|
}
|
||||||
|
// Scale with the project when the caller knows the DB size: every fold
|
||||||
|
// re-writes hot B-tree pages into the main file (the #1231 pathology in
|
||||||
|
// bounded form — 111s of a kernel-scale batch loop at the flat 256MB cap,
|
||||||
|
// §7a.2), so a big project affords a proportionally bigger transient WAL
|
||||||
|
// (~dbSize/4 soft ⇒ file cap ≈ dbSize) in exchange for ~4× fewer folds.
|
||||||
|
if (dbSizeBytes !== undefined && dbSizeBytes > 0) {
|
||||||
|
return Math.min(2048, Math.max(DEFAULT_WAL_VALVE_MB, Math.floor(dbSizeBytes / 4 / (1024 * 1024))));
|
||||||
|
}
|
||||||
return DEFAULT_WAL_VALVE_MB;
|
return DEFAULT_WAL_VALVE_MB;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -25,7 +25,7 @@ import {
|
|||||||
FindRelevantContextOptions,
|
FindRelevantContextOptions,
|
||||||
} from './types';
|
} from './types';
|
||||||
import { DatabaseConnection, getDatabasePath, removeDatabaseFiles } from './db';
|
import { DatabaseConnection, getDatabasePath, removeDatabaseFiles } from './db';
|
||||||
import { WalCheckpointValve } from './db/wal-valve';
|
import { WalCheckpointValve, resolveWalValveMb } from './db/wal-valve';
|
||||||
import { QueryBuilder } from './db/queries';
|
import { QueryBuilder } from './db/queries';
|
||||||
import {
|
import {
|
||||||
isInitialized,
|
isInitialized,
|
||||||
@@ -472,7 +472,7 @@ export class CodeGraph {
|
|||||||
this.db.setWalAutocheckpoint(0);
|
this.db.setWalAutocheckpoint(0);
|
||||||
walValve = new WalCheckpointValve(
|
walValve = new WalCheckpointValve(
|
||||||
this.db,
|
this.db,
|
||||||
undefined,
|
resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB, this.db.getDbFileSizeBytes()),
|
||||||
undefined,
|
undefined,
|
||||||
options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
|
options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
|
||||||
);
|
);
|
||||||
@@ -753,7 +753,7 @@ export class CodeGraph {
|
|||||||
this.db.setWalAutocheckpoint(0);
|
this.db.setWalAutocheckpoint(0);
|
||||||
walValve = new WalCheckpointValve(
|
walValve = new WalCheckpointValve(
|
||||||
this.db,
|
this.db,
|
||||||
undefined,
|
resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB, this.db.getDbFileSizeBytes()),
|
||||||
undefined,
|
undefined,
|
||||||
options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
|
options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
|
||||||
);
|
);
|
||||||
|
|||||||
+101
-14
@@ -637,7 +637,7 @@ export class ReferenceResolver {
|
|||||||
|
|
||||||
for (let i = 0; i < refs.length; i++) {
|
for (let i = 0; i < refs.length; i++) {
|
||||||
const ref = refs[i]!; // Array index is guaranteed to be in bounds
|
const ref = refs[i]!; // Array index is guaranteed to be in bounds
|
||||||
const result = this.resolveOne(ref);
|
const result = this.resolveOneTimed(ref);
|
||||||
|
|
||||||
if (result) {
|
if (result) {
|
||||||
resolved.push(result);
|
resolved.push(result);
|
||||||
@@ -1233,7 +1233,7 @@ export class ReferenceResolver {
|
|||||||
language: raw.language || this.getLanguageFromNodeId(raw.fromNodeId),
|
language: raw.language || this.getLanguageFromNodeId(raw.fromNodeId),
|
||||||
rowId: raw.rowId,
|
rowId: raw.rowId,
|
||||||
};
|
};
|
||||||
const result = this.resolveOne(ref);
|
const result = this.resolveOneTimed(ref);
|
||||||
if (result) {
|
if (result) {
|
||||||
resolved.push(result);
|
resolved.push(result);
|
||||||
byMethod[result.resolvedBy] = (byMethod[result.resolvedBy] || 0) + 1;
|
byMethod[result.resolvedBy] = (byMethod[result.resolvedBy] || 0) + 1;
|
||||||
@@ -1266,6 +1266,47 @@ export class ReferenceResolver {
|
|||||||
* of resolveBatchYielding, minus the main-thread yields (worker threads have
|
* of resolveBatchYielding, minus the main-thread yields (worker threads have
|
||||||
* no watchdog heartbeat to starve). Results are in input order.
|
* no watchdog heartbeat to starve). Results are in input order.
|
||||||
*/
|
*/
|
||||||
|
/**
|
||||||
|
* CODEGRAPH_RESOLVE_PROFILE=1: per-outcome wall-clock histogram of
|
||||||
|
* resolveOne, keyed by the winning strategy (`resolvedBy`) or
|
||||||
|
* `fail:<referenceKind>` — the §7a.2 "profile the per-ref path" probe. The
|
||||||
|
* kernel-scale batch loop is ~430s and CORE-INVARIANT (835.9s pooled-4-on-8
|
||||||
|
* ≈ 812.5s sequential-on-2 for the whole superphase), so the next lever is
|
||||||
|
* which CLASS of ref the time belongs to, not more parallelism. Off by
|
||||||
|
* default: the hrtime pair costs ~100ns/ref only when the env is set.
|
||||||
|
*/
|
||||||
|
private resolveProfile: Map<string, { n: number; ns: bigint }> | null =
|
||||||
|
process.env.CODEGRAPH_RESOLVE_PROFILE ? new Map() : null;
|
||||||
|
|
||||||
|
private resolveOneTimed(ref: UnresolvedRef): ResolvedRef | null {
|
||||||
|
if (!this.resolveProfile) return this.resolveOne(ref);
|
||||||
|
const t0 = process.hrtime.bigint();
|
||||||
|
const result = this.resolveOne(ref);
|
||||||
|
const dt = process.hrtime.bigint() - t0;
|
||||||
|
const key = result ? result.resolvedBy : `fail:${ref.referenceKind}`;
|
||||||
|
const slot = this.resolveProfile.get(key);
|
||||||
|
if (slot) {
|
||||||
|
slot.n++;
|
||||||
|
slot.ns += dt;
|
||||||
|
} else {
|
||||||
|
this.resolveProfile.set(key, { n: 1, ns: dt });
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Dump the CODEGRAPH_RESOLVE_PROFILE histogram to stderr (no-op when off). */
|
||||||
|
dumpResolveProfile(label: string): void {
|
||||||
|
if (!this.resolveProfile || this.resolveProfile.size === 0) return;
|
||||||
|
const rows = [...this.resolveProfile.entries()]
|
||||||
|
.map(([k, v]) => ({ k, n: v.n, ms: Number(v.ns / 1_000_000n) }))
|
||||||
|
.sort((a, b) => b.ms - a.ms);
|
||||||
|
for (const r of rows) {
|
||||||
|
console.error(
|
||||||
|
`[resolve-profile] ${label} ${r.k}: n=${r.n} total=${(r.ms / 1000).toFixed(1)}s avg=${((r.ms * 1000) / Math.max(1, r.n)).toFixed(0)}µs`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
resolveListForAdmission(refs: UnresolvedReference[]): {
|
resolveListForAdmission(refs: UnresolvedReference[]): {
|
||||||
resolved: ResolvedRef[];
|
resolved: ResolvedRef[];
|
||||||
unresolved: UnresolvedRef[];
|
unresolved: UnresolvedRef[];
|
||||||
@@ -1288,7 +1329,7 @@ export class ReferenceResolver {
|
|||||||
language: raw.language || this.getLanguageFromNodeId(raw.fromNodeId),
|
language: raw.language || this.getLanguageFromNodeId(raw.fromNodeId),
|
||||||
rowId: raw.rowId,
|
rowId: raw.rowId,
|
||||||
};
|
};
|
||||||
const result = this.resolveOne(ref);
|
const result = this.resolveOneTimed(ref);
|
||||||
if (result) {
|
if (result) {
|
||||||
resolved.push(result);
|
resolved.push(result);
|
||||||
byMethod[result.resolvedBy] = (byMethod[result.resolvedBy] || 0) + 1;
|
byMethod[result.resolvedBy] = (byMethod[result.resolvedBy] || 0) + 1;
|
||||||
@@ -1362,6 +1403,16 @@ export class ReferenceResolver {
|
|||||||
console.error(`[pool-timing] backpressure hook: ${parallel?.backpressure ? 'present' : 'absent'}`);
|
console.error(`[pool-timing] backpressure hook: ${parallel?.backpressure ? 'present' : 'absent'}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CODEGRAPH_RESOLVE_PROFILE loop-stage attribution: the §7a.2 kernel-scale
|
||||||
|
// histogram showed resolveOne owns only ~93s of the ~436s batch loop —
|
||||||
|
// these counters name where the other ~340s goes (reads, edge build+insert,
|
||||||
|
// deletes/marks, the per-batch count guard).
|
||||||
|
const loopProf: Record<string, number> | null = process.env.CODEGRAPH_RESOLVE_PROFILE
|
||||||
|
? { read: 0, settle: 0, backpressure: 0, createEdges: 0, insertEdges: 0, deletes: 0, marks: 0, countGuard: 0 }
|
||||||
|
: null;
|
||||||
|
const lp = (k: string, t0: number): void => { if (loopProf) loopProf[k] = (loopProf[k] ?? 0) + (Date.now() - t0); };
|
||||||
|
let tLp = 0;
|
||||||
|
|
||||||
await this.warmCachesYielding(maybeYield);
|
await this.warmCachesYielding(maybeYield);
|
||||||
|
|
||||||
const total = this.queries.getUnresolvedReferencesCount();
|
const total = this.queries.getUnresolvedReferencesCount();
|
||||||
@@ -1476,18 +1527,24 @@ export class ReferenceResolver {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
try {
|
try {
|
||||||
let batch = this.queries.getUnresolvedReferencesBatch(0, batchSize);
|
tLp = Date.now();
|
||||||
|
let batch = this.queries.getUnresolvedReferencesBatchAfter(0, batchSize);
|
||||||
|
lp('read', tLp);
|
||||||
let inFlight: InFlight | null = batch.length > 0 ? beginBatch(batch) : null;
|
let inFlight: InFlight | null = batch.length > 0 ? beginBatch(batch) : null;
|
||||||
while (batch.length > 0 && inFlight) {
|
while (batch.length > 0 && inFlight) {
|
||||||
// Prefetch the NEXT batch before this one persists: this batch's rows
|
// Prefetch the NEXT batch before this one persists: this batch's rows
|
||||||
// are still pending (nothing has mutated the table since they were
|
// are still pending (nothing has mutated the table since they were
|
||||||
// read), so skipping exactly batch.length rows in the same rowid
|
// read), so seeking past this batch's last row id in the same rowid
|
||||||
// enumeration yields the following batch.
|
// enumeration yields the following batch (keyset — OFFSET re-walked the
|
||||||
const nextBatch = this.queries.getUnresolvedReferencesBatch(batch.length, batchSize);
|
// accumulated failed prefix every read, 54.6s at kernel scale, §7a.2).
|
||||||
|
tLp = Date.now();
|
||||||
|
const nextBatch = this.queries.getUnresolvedReferencesBatchAfter(batch[batch.length - 1]!.rowId!, batchSize);
|
||||||
|
lp('read', tLp);
|
||||||
|
|
||||||
const tBatch = Date.now();
|
const tBatch = Date.now();
|
||||||
const result = await settleBatch(inFlight, batch);
|
const result = await settleBatch(inFlight, batch);
|
||||||
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[pool-timing] batch ${inFlight.mode}: ${batch.length} refs in ${Date.now() - tBatch}ms`);
|
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[pool-timing] batch ${inFlight.mode}: ${batch.length} refs in ${Date.now() - tBatch}ms`);
|
||||||
|
lp('settle', tBatch);
|
||||||
|
|
||||||
// WAL-valve backstop at the ONE pool-idle boundary of the double-buffer
|
// WAL-valve backstop at the ONE pool-idle boundary of the double-buffer
|
||||||
// (this batch settled, the next not yet fanned out): past the hard cap
|
// (this batch settled, the next not yet fanned out): past the hard cap
|
||||||
@@ -1495,8 +1552,10 @@ export class ReferenceResolver {
|
|||||||
// are all between statements — so the backfill completes, readers
|
// are all between statements — so the backfill completes, readers
|
||||||
// re-enter at SQLite's backfilled mark, and the next persist commit
|
// re-enter at SQLite's backfilled mark, and the next persist commit
|
||||||
// WRAPS the WAL instead of growing it. No-op (one fstat) under the cap.
|
// WRAPS the WAL instead of growing it. No-op (one fstat) under the cap.
|
||||||
|
tLp = Date.now();
|
||||||
const bp = parallel?.backpressure?.();
|
const bp = parallel?.backpressure?.();
|
||||||
if (bp) await bp;
|
if (bp) await bp;
|
||||||
|
lp('backpressure', tLp);
|
||||||
|
|
||||||
// Persist in bounded sub-transactions with yields between: a whole
|
// Persist in bounded sub-transactions with yields between: a whole
|
||||||
// batch's edge insert / keyed deletes are otherwise one solid
|
// batch's edge insert / keyed deletes are otherwise one solid
|
||||||
@@ -1515,11 +1574,15 @@ export class ReferenceResolver {
|
|||||||
// base class if those edges are visible. (Validated on dubbo: fanning
|
// base class if those edges are visible. (Validated on dubbo: fanning
|
||||||
// out first downgraded exactly those supertype-method resolutions from
|
// out first downgraded exactly those supertype-method resolutions from
|
||||||
// the 0.9 typed-receiver path to the 0.65 word-overlap fallback.)
|
// the 0.9 typed-receiver path to the 0.65 word-overlap fallback.)
|
||||||
|
tLp = Date.now();
|
||||||
const edges = this.createEdges(result.resolved);
|
const edges = this.createEdges(result.resolved);
|
||||||
|
lp('createEdges', tLp);
|
||||||
|
tLp = Date.now();
|
||||||
for (let i = 0; i < edges.length; i += PERSIST_CHUNK) {
|
for (let i = 0; i < edges.length; i += PERSIST_CHUNK) {
|
||||||
this.queries.insertEdges(edges.slice(i, i + PERSIST_CHUNK));
|
this.queries.insertEdges(edges.slice(i, i + PERSIST_CHUNK));
|
||||||
await maybeYield();
|
await maybeYield();
|
||||||
}
|
}
|
||||||
|
lp('insertEdges', tLp);
|
||||||
|
|
||||||
// NOW fan the next batch out — workers see exactly the edge state the
|
// NOW fan the next batch out — workers see exactly the edge state the
|
||||||
// sequential baseline would (every batch ≤ this one committed), while
|
// sequential baseline would (every batch ≤ this one committed), while
|
||||||
@@ -1531,29 +1594,34 @@ export class ReferenceResolver {
|
|||||||
// by row id, so a same-key sibling ref in a LATER batch (same caller
|
// by row id, so a same-key sibling ref in a LATER batch (same caller
|
||||||
// calling the same callee at another line) is left pending for its own
|
// calling the same callee at another line) is left pending for its own
|
||||||
// attempt instead of being swept out with this batch's rows (#1269).
|
// attempt instead of being swept out with this batch's rows (#1269).
|
||||||
|
tLp = Date.now();
|
||||||
|
let removedThisBatch = 0;
|
||||||
const resolvedCleanup = ReferenceResolver.partitionResolvedCleanup(result.resolved);
|
const resolvedCleanup = ReferenceResolver.partitionResolvedCleanup(result.resolved);
|
||||||
for (let i = 0; i < resolvedCleanup.rowIds.length; i += PERSIST_CHUNK) {
|
for (let i = 0; i < resolvedCleanup.rowIds.length; i += PERSIST_CHUNK) {
|
||||||
this.queries.deleteReferencesByRowIds(resolvedCleanup.rowIds.slice(i, i + PERSIST_CHUNK));
|
removedThisBatch += this.queries.deleteReferencesByRowIds(resolvedCleanup.rowIds.slice(i, i + PERSIST_CHUNK));
|
||||||
await maybeYield();
|
await maybeYield();
|
||||||
}
|
}
|
||||||
for (let i = 0; i < resolvedCleanup.legacyKeys.length; i += PERSIST_CHUNK) {
|
for (let i = 0; i < resolvedCleanup.legacyKeys.length; i += PERSIST_CHUNK) {
|
||||||
this.queries.deleteSpecificResolvedReferences(resolvedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
removedThisBatch += this.queries.deleteSpecificResolvedReferences(resolvedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
||||||
await maybeYield();
|
await maybeYield();
|
||||||
}
|
}
|
||||||
|
lp('deletes', tLp);
|
||||||
|
|
||||||
// Park unresolvable refs from this batch as status='failed' so they
|
// Park unresolvable refs from this batch as status='failed' so they
|
||||||
// leave the pending set (the batch reader and non-progress guard below
|
// leave the pending set (the batch reader and non-progress guard below
|
||||||
// only see pending rows) but stay retryable when a later sync adds a
|
// only see pending rows) but stay retryable when a later sync adds a
|
||||||
// symbol that could satisfy them (#1240).
|
// symbol that could satisfy them (#1240).
|
||||||
|
tLp = Date.now();
|
||||||
const failedCleanup = ReferenceResolver.partitionFailedCleanup(result.unresolved);
|
const failedCleanup = ReferenceResolver.partitionFailedCleanup(result.unresolved);
|
||||||
for (let i = 0; i < failedCleanup.byRowId.length; i += PERSIST_CHUNK) {
|
for (let i = 0; i < failedCleanup.byRowId.length; i += PERSIST_CHUNK) {
|
||||||
this.queries.markReferencesFailedByRowIds(failedCleanup.byRowId.slice(i, i + PERSIST_CHUNK));
|
removedThisBatch += this.queries.markReferencesFailedByRowIds(failedCleanup.byRowId.slice(i, i + PERSIST_CHUNK));
|
||||||
await maybeYield();
|
await maybeYield();
|
||||||
}
|
}
|
||||||
for (let i = 0; i < failedCleanup.legacyKeys.length; i += PERSIST_CHUNK) {
|
for (let i = 0; i < failedCleanup.legacyKeys.length; i += PERSIST_CHUNK) {
|
||||||
this.queries.markReferencesFailed(failedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
removedThisBatch += this.queries.markReferencesFailed(failedCleanup.legacyKeys.slice(i, i + PERSIST_CHUNK));
|
||||||
await maybeYield();
|
await maybeYield();
|
||||||
}
|
}
|
||||||
|
lp('marks', tLp);
|
||||||
|
|
||||||
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[pool-timing] batch persist: ${Date.now() - tPersist}ms`);
|
if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[pool-timing] batch persist: ${Date.now() - tPersist}ms`);
|
||||||
|
|
||||||
@@ -1590,9 +1658,22 @@ export class ReferenceResolver {
|
|||||||
// 1.4 GB before the Go-fallback fix). Stop rather than grow the graph
|
// 1.4 GB before the Go-fallback fix). Stop rather than grow the graph
|
||||||
// without bound. (An in-flight prefetched batch is abandoned unsettled —
|
// without bound. (An in-flight prefetched batch is abandoned unsettled —
|
||||||
// fan-out has no side effects until settleBatch appends its results.)
|
// fan-out has no side effects until settleBatch appends its results.)
|
||||||
const remaining = this.queries.getUnresolvedReferencesCount();
|
// Non-progress signal, now O(1): `changes` summed across this batch's
|
||||||
if (remaining >= prevRemaining) break;
|
// deletes + failed-parks is the DIRECT evidence the guard's old count
|
||||||
prevRemaining = remaining;
|
// diff inferred — a resolver returning a mismatched name makes the keyed
|
||||||
|
// cleanup no-op, which shows up here as zero removals. The per-batch
|
||||||
|
// COUNT(*) it replaces walked every remaining pending row — O(N²/batch)
|
||||||
|
// over a run, 93.9s of the kernel-scale batch loop (§7a.2). A REAL count
|
||||||
|
// runs only on the suspicious path (claimed-work batch removed nothing —
|
||||||
|
// e.g. every row was a sibling a legacy-key sweep already consumed),
|
||||||
|
// where it arbitrates stop-vs-continue exactly as before.
|
||||||
|
if (removedThisBatch <= 0 && batch.length > 0) {
|
||||||
|
tLp = Date.now();
|
||||||
|
const remaining = this.queries.getUnresolvedReferencesCount();
|
||||||
|
lp('countGuard', tLp);
|
||||||
|
if (remaining >= prevRemaining) break;
|
||||||
|
prevRemaining = remaining;
|
||||||
|
}
|
||||||
|
|
||||||
// Advance the pipeline: the prefetched batch (already fanned out when
|
// Advance the pipeline: the prefetched batch (already fanned out when
|
||||||
// the pool is on) becomes the current one.
|
// the pool is on) becomes the current one.
|
||||||
@@ -1639,6 +1720,12 @@ export class ReferenceResolver {
|
|||||||
if (pool) await pool.destroy().catch(() => undefined);
|
if (pool) await pool.destroy().catch(() => undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (loopProf) {
|
||||||
|
const parts = Object.entries(loopProf).map(([k, v]) => `${k}=${(v / 1000).toFixed(1)}s`).join(' ');
|
||||||
|
console.error(`[resolve-profile] loop-stages ${parts}`);
|
||||||
|
}
|
||||||
|
this.dumpResolveProfile('main');
|
||||||
|
|
||||||
return {
|
return {
|
||||||
resolved: [],
|
resolved: [],
|
||||||
unresolved: [],
|
unresolved: [],
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ try {
|
|||||||
(require('node:module') as { enableCompileCache?: () => void }).enableCompileCache?.();
|
(require('node:module') as { enableCompileCache?: () => void }).enableCompileCache?.();
|
||||||
} catch { /* cache is best-effort */ }
|
} catch { /* cache is best-effort */ }
|
||||||
|
|
||||||
import { parentPort } from 'worker_threads';
|
import { parentPort, threadId } from 'worker_threads';
|
||||||
import { createDatabase, SqliteDatabase } from '../db/sqlite-adapter';
|
import { createDatabase, SqliteDatabase } from '../db/sqlite-adapter';
|
||||||
import { QueryBuilder } from '../db/queries';
|
import { QueryBuilder } from '../db/queries';
|
||||||
import { ReferenceResolver } from './index';
|
import { ReferenceResolver } from './index';
|
||||||
@@ -95,6 +95,9 @@ port.on('message', (msg: InMessage) => {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case 'close': {
|
case 'close': {
|
||||||
|
try {
|
||||||
|
resolver?.dumpResolveProfile(`worker#${threadId}`);
|
||||||
|
} catch { /* diagnostics never block shutdown */ }
|
||||||
try {
|
try {
|
||||||
db?.close();
|
db?.close();
|
||||||
} catch {
|
} catch {
|
||||||
|
|||||||
Reference in New Issue
Block a user