feat: Add database schema v3 with optimized node lookups and improved error handling
Adds expression index on lower(name) for memory-efficient case-insensitive searches, replacing in-memory caches that caused OOM on large codebases. Includes batched reference resolution, enhanced error reporting with detailed breakdown by error type, and improved CLI progress display for scanning phases.
This commit is contained in:
Colby McHenry
+92
-85
@@ -34,13 +34,8 @@ export class ReferenceResolver {
|
||||
private queries: QueryBuilder;
|
||||
private context: ResolutionContext;
|
||||
private frameworks: FrameworkResolver[] = [];
|
||||
private nodeCache: Map<string, Node[]> = new Map();
|
||||
private fileCache: Map<string, string | null> = new Map();
|
||||
private nameCache: Map<string, Node[]> = new Map();
|
||||
private qualifiedNameCache: Map<string, Node[]> = new Map();
|
||||
private kindCache: Map<string, Node[]> = new Map();
|
||||
private nodeByIdCache: Map<string, Node> = new Map();
|
||||
private lowerNameCache: Map<string, Node[]> = new Map();
|
||||
private nodeCache: Map<string, Node[]> = new Map(); // per-file node cache (bounded)
|
||||
private fileCache: Map<string, string | null> = new Map(); // per-file content cache (bounded)
|
||||
private importMappingCache: Map<string, ImportMapping[]> = new Map();
|
||||
private knownFiles: Set<string> | null = null;
|
||||
private cachesWarmed = false;
|
||||
@@ -60,53 +55,15 @@ export class ReferenceResolver {
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-load all nodes into memory maps for fast lookup during resolution.
|
||||
* This eliminates repeated SQLite queries and provides the core speedup.
|
||||
* Pre-build lightweight caches for resolution.
|
||||
* Node lookups are now handled by indexed SQLite queries instead of
|
||||
* loading all nodes into memory (which caused OOM on large codebases).
|
||||
*/
|
||||
warmCaches(): void {
|
||||
if (this.cachesWarmed) return;
|
||||
|
||||
const allNodes = this.queries.getAllNodes();
|
||||
for (const node of allNodes) {
|
||||
// Index by name
|
||||
const byName = this.nameCache.get(node.name);
|
||||
if (byName) {
|
||||
byName.push(node);
|
||||
} else {
|
||||
this.nameCache.set(node.name, [node]);
|
||||
}
|
||||
|
||||
// Index by qualified name
|
||||
const byQName = this.qualifiedNameCache.get(node.qualifiedName);
|
||||
if (byQName) {
|
||||
byQName.push(node);
|
||||
} else {
|
||||
this.qualifiedNameCache.set(node.qualifiedName, [node]);
|
||||
}
|
||||
|
||||
// Index by kind
|
||||
const byKind = this.kindCache.get(node.kind);
|
||||
if (byKind) {
|
||||
byKind.push(node);
|
||||
} else {
|
||||
this.kindCache.set(node.kind, [node]);
|
||||
}
|
||||
|
||||
// Index by ID
|
||||
this.nodeByIdCache.set(node.id, node);
|
||||
|
||||
// Index by lowercase name (for fuzzy matching)
|
||||
const lowerName = node.name.toLowerCase();
|
||||
const byLower = this.lowerNameCache.get(lowerName);
|
||||
if (byLower) {
|
||||
byLower.push(node);
|
||||
} else {
|
||||
this.lowerNameCache.set(lowerName, [node]);
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-build known files set from index
|
||||
this.knownFiles = new Set(this.queries.getAllFiles().map((f) => f.path));
|
||||
// Only cache the set of known file paths (lightweight string set)
|
||||
this.knownFiles = new Set(this.queries.getAllFilePaths());
|
||||
|
||||
this.cachesWarmed = true;
|
||||
}
|
||||
@@ -117,11 +74,6 @@ export class ReferenceResolver {
|
||||
clearCaches(): void {
|
||||
this.nodeCache.clear();
|
||||
this.fileCache.clear();
|
||||
this.nameCache.clear();
|
||||
this.qualifiedNameCache.clear();
|
||||
this.kindCache.clear();
|
||||
this.nodeByIdCache.clear();
|
||||
this.lowerNameCache.clear();
|
||||
this.importMappingCache.clear();
|
||||
this.knownFiles = null;
|
||||
this.cachesWarmed = false;
|
||||
@@ -140,28 +92,14 @@ export class ReferenceResolver {
|
||||
},
|
||||
|
||||
getNodesByName: (name: string) => {
|
||||
// Use warm cache if available, otherwise fall back to search
|
||||
if (this.cachesWarmed) {
|
||||
return this.nameCache.get(name) ?? [];
|
||||
}
|
||||
return this.queries.searchNodes(name, { limit: 100 }).map((r) => r.node);
|
||||
return this.queries.getNodesByName(name);
|
||||
},
|
||||
|
||||
getNodesByQualifiedName: (qualifiedName: string) => {
|
||||
// Use warm cache if available, otherwise fall back to search + filter
|
||||
if (this.cachesWarmed) {
|
||||
return this.qualifiedNameCache.get(qualifiedName) ?? [];
|
||||
}
|
||||
return this.queries
|
||||
.searchNodes(qualifiedName, { limit: 50 })
|
||||
.filter((r) => r.node.qualifiedName === qualifiedName)
|
||||
.map((r) => r.node);
|
||||
return this.queries.getNodesByQualifiedNameExact(qualifiedName);
|
||||
},
|
||||
|
||||
getNodesByKind: (kind: Node['kind']) => {
|
||||
if (this.cachesWarmed) {
|
||||
return this.kindCache.get(kind) ?? [];
|
||||
}
|
||||
return this.queries.getNodesByKind(kind);
|
||||
},
|
||||
|
||||
@@ -203,17 +141,11 @@ export class ReferenceResolver {
|
||||
getProjectRoot: () => this.projectRoot,
|
||||
|
||||
getAllFiles: () => {
|
||||
return this.queries.getAllFiles().map((f) => f.path);
|
||||
return this.queries.getAllFilePaths();
|
||||
},
|
||||
|
||||
getNodesByLowerName: (lowerName: string) => {
|
||||
if (this.cachesWarmed) {
|
||||
return this.lowerNameCache.get(lowerName) ?? [];
|
||||
}
|
||||
// Fallback: scan all nodes (expensive, but only used if cache not warm)
|
||||
return this.queries.getAllNodes().filter(
|
||||
(n) => n.name.toLowerCase() === lowerName
|
||||
);
|
||||
return this.queries.getNodesByLowerName(lowerName);
|
||||
},
|
||||
|
||||
getImportMappings: (filePath: string, language) => {
|
||||
@@ -389,6 +321,87 @@ export class ReferenceResolver {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve and persist in batches to keep memory bounded.
|
||||
* Processes unresolved references in chunks, persisting edges and cleaning
|
||||
* up resolved refs after each batch to avoid accumulating large arrays.
|
||||
*/
|
||||
resolveAndPersistBatched(
|
||||
onProgress?: (current: number, total: number) => void,
|
||||
batchSize: number = 5000
|
||||
): ResolutionResult {
|
||||
this.warmCaches();
|
||||
|
||||
const total = this.queries.getUnresolvedReferencesCount();
|
||||
let processed = 0;
|
||||
const aggregateStats = {
|
||||
total: 0,
|
||||
resolved: 0,
|
||||
unresolved: 0,
|
||||
byMethod: {} as Record<string, number>,
|
||||
};
|
||||
|
||||
// Process in batches. We always read from offset 0 because resolved refs
|
||||
// are deleted after each batch, shifting the remaining rows forward.
|
||||
while (true) {
|
||||
const batch = this.queries.getUnresolvedReferencesBatch(0, batchSize);
|
||||
if (batch.length === 0) break;
|
||||
|
||||
const result = this.resolveAll(batch);
|
||||
|
||||
// Persist edges immediately
|
||||
const edges = this.createEdges(result.resolved);
|
||||
if (edges.length > 0) {
|
||||
this.queries.insertEdges(edges);
|
||||
}
|
||||
|
||||
// Clean up resolved refs so they don't appear in the next batch
|
||||
if (result.resolved.length > 0) {
|
||||
this.queries.deleteSpecificResolvedReferences(
|
||||
result.resolved.map((r) => ({
|
||||
fromNodeId: r.original.fromNodeId,
|
||||
referenceName: r.original.referenceName,
|
||||
referenceKind: r.original.referenceKind,
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
// Delete unresolvable refs from this batch to avoid re-processing them
|
||||
if (result.unresolved.length > 0) {
|
||||
this.queries.deleteSpecificResolvedReferences(
|
||||
result.unresolved.map((r) => ({
|
||||
fromNodeId: r.fromNodeId,
|
||||
referenceName: r.referenceName,
|
||||
referenceKind: r.referenceKind,
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
// Aggregate stats
|
||||
aggregateStats.total += result.stats.total;
|
||||
aggregateStats.resolved += result.stats.resolved;
|
||||
aggregateStats.unresolved += result.stats.unresolved;
|
||||
for (const [method, count] of Object.entries(result.stats.byMethod)) {
|
||||
aggregateStats.byMethod[method] = (aggregateStats.byMethod[method] || 0) + count;
|
||||
}
|
||||
|
||||
processed += batch.length;
|
||||
onProgress?.(processed, total);
|
||||
|
||||
// If nothing was resolved or removed in this batch, we'd loop forever
|
||||
// on the same rows. Break to avoid infinite loop.
|
||||
if (result.resolved.length === 0 && result.unresolved.length === batch.length) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
resolved: [],
|
||||
unresolved: [],
|
||||
stats: aggregateStats,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get detected frameworks
|
||||
*/
|
||||
@@ -513,9 +526,6 @@ export class ReferenceResolver {
|
||||
* Get file path from node ID
|
||||
*/
|
||||
private getFilePathFromNodeId(nodeId: string): string {
|
||||
// Check warm cache first
|
||||
const cached = this.nodeByIdCache.get(nodeId);
|
||||
if (cached) return cached.filePath;
|
||||
const node = this.queries.getNodeById(nodeId);
|
||||
return node?.filePath || '';
|
||||
}
|
||||
@@ -524,9 +534,6 @@ export class ReferenceResolver {
|
||||
* Get language from node ID
|
||||
*/
|
||||
private getLanguageFromNodeId(nodeId: string): UnresolvedRef['language'] {
|
||||
// Check warm cache first
|
||||
const cached = this.nodeByIdCache.get(nodeId);
|
||||
if (cached) return cached.language;
|
||||
const node = this.queries.getNodeById(nodeId);
|
||||
return node?.language || 'unknown';
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user