From 1271ad9161cbc900bbc7bab41bde300fa51c09c9 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Sat, 4 Apr 2026 22:49:36 -0500 Subject: [PATCH] feat: Add adaptive timeouts and WASM memory error recovery for robust parsing Implements file-size-based timeouts (base 10s + 10s per 100KB), more frequent worker recycling (250 files), and automatic retry logic for WASM memory corruption failures. Workers now crash immediately on memory errors to prevent cascading failures, with failed files automatically retried on fresh workers with clean heaps. --- src/extraction/index.ts | 71 +++++++++++++++++++++++++++++++--- src/extraction/parse-worker.ts | 8 ++++ src/extraction/tree-sitter.ts | 11 +++++- 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 29e366c..355bf8d 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -45,7 +45,7 @@ const PARSE_TIMEOUT_MS = 10_000; * V8 isolate by terminating the worker thread and spawning a fresh one. * This interval balances memory usage against the cost of reloading grammars. */ -const WORKER_RECYCLE_INTERVAL = 500; +const WORKER_RECYCLE_INTERVAL = 250; /** * Progress callback for indexing operations @@ -521,8 +521,12 @@ export class ExtractionOrchestrator { logWarn('Parse worker exited unexpectedly', { code }); rejectAllPending(`Worker exited with code ${code}`); } - // Clear reference so we know to respawn - if (parseWorker === w) parseWorker = null; + // Clear reference so we know to respawn, reset count so + // the fresh worker gets a full cycle before recycling. + if (parseWorker === w) { + parseWorker = null; + workerParseCount = 0; + } }); } @@ -580,17 +584,20 @@ export class ExtractionOrchestrator { const id = nextId++; workerParseCount++; + // Scale timeout for large files: base 10s + 10s per 100KB + const timeoutMs = PARSE_TIMEOUT_MS + Math.floor(content.length / 100_000) * 10_000; + return new Promise((resolve, reject) => { const timer = setTimeout(() => { pendingParses.delete(id); - log(`TIMEOUT: ${filePath} exceeded ${PARSE_TIMEOUT_MS}ms — killing worker`); + log(`TIMEOUT: ${filePath} exceeded ${timeoutMs}ms — killing worker`); // Reject FIRST — worker.terminate() can hang if WASM is stuck parseWorker = null; workerParseCount = 0; - reject(new Error(`Parse timed out after ${PARSE_TIMEOUT_MS}ms`)); + reject(new Error(`Parse timed out after ${timeoutMs}ms`)); // Fire-and-forget: kill the stuck worker in the background worker.terminate().catch(() => {}); - }, PARSE_TIMEOUT_MS); + }, timeoutMs); pendingParses.set(id, { resolve, reject, timer }); worker.postMessage({ type: 'parse', id, filePath, content }); @@ -712,6 +719,58 @@ export class ExtractionOrchestrator { } } + // Retry pass: files that failed due to WASM memory corruption may succeed + // on a fresh worker with a clean heap. Collect retryable failures, recycle + // the worker, and try each one individually. + const retryableErrors = errors.filter( + (e) => e.code === 'parse_error' && e.filePath && + (e.message.includes('Worker exited') || e.message.includes('memory access out of bounds')) + ); + + if (retryableErrors.length > 0 && WorkerClass) { + log(`Retrying ${retryableErrors.length} files that failed due to WASM memory errors...`); + + // Force a fresh worker + recycleWorker(); + + for (const errEntry of retryableErrors) { + const filePath = errEntry.filePath!; + if (signal?.aborted) break; + + let content: string; + try { + const fullPath = validatePathWithinRoot(this.rootDir, filePath); + if (!fullPath) continue; + content = await fsp.readFile(fullPath, 'utf-8'); + } catch { + continue; // Skip files we can't read + } + + let result: ExtractionResult; + try { + result = await requestParse(filePath, content); + } catch { + continue; // Still failing — leave as errored + } + + if (result.nodes.length > 0 || result.errors.length === 0) { + // Success on retry — store result and fix counts + const language = detectLanguage(filePath); + const stats = await fsp.stat(path.join(this.rootDir, filePath)); + this.storeExtractionResult(filePath, content, language, stats, result); + + // Remove the original error and update counts + const idx = errors.indexOf(errEntry); + if (idx >= 0) errors.splice(idx, 1); + filesErrored--; + filesIndexed++; + totalNodes += result.nodes.length; + totalEdges += result.edges.length; + log(`Retry OK: ${filePath} (${result.nodes.length} nodes)`); + } + } + } + // Shut down parse worker and clear any pending timers rejectAllPending('Indexing complete'); if (parseWorker) { diff --git a/src/extraction/parse-worker.ts b/src/extraction/parse-worker.ts index f8e7dec..f4235e4 100644 --- a/src/extraction/parse-worker.ts +++ b/src/extraction/parse-worker.ts @@ -33,6 +33,14 @@ parentPort!.on('message', async (msg: { type: string; id?: number; filePath?: st parentPort!.postMessage({ type: 'parse-result', id, result }); } catch (err) { const message = err instanceof Error ? err.message : String(err); + + // WASM memory errors leave the module in a corrupted state — all + // subsequent parses would also fail (cascading failures). Crash the + // worker so the main thread spawns a fresh one with a clean heap. + if (message.includes('memory access out of bounds') || message.includes('out of memory')) { + process.exit(1); + } + parentPort!.postMessage({ type: 'parse-result', id, diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index ab2d3df..9c7cf38 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -172,8 +172,17 @@ export class TreeSitterExtractor { this.visitNode(this.tree.rootNode); this.nodeStack.pop(); } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + + // WASM memory errors leave the module in a corrupted state — all subsequent + // parses would also fail. Re-throw so the worker can detect and crash, + // forcing a clean restart with a fresh heap. + if (msg.includes('memory access out of bounds') || msg.includes('out of memory')) { + throw error; + } + this.errors.push({ - message: `Parse error: ${error instanceof Error ? error.message : String(error)}`, + message: `Parse error: ${msg}`, filePath: this.filePath, severity: 'error', code: 'parse_error',