fix: Improve CLI progress display and prevent tree-sitter WASM memory crashes
Replaces fixed-width padding with terminal escape sequences for proper progress line clearing across different terminal widths. Adds periodic parser reset every 5000 parses per language to prevent WASM heap fragmentation that causes "memory access out of bounds" crashes in large repositories. Includes filename truncation to fit available terminal width.
This commit is contained in:
+18
-1
@@ -18,7 +18,7 @@ import {
|
||||
} from '../types';
|
||||
import { QueryBuilder } from '../db/queries';
|
||||
import { extractFromSource } from './tree-sitter';
|
||||
import { detectLanguage, isLanguageSupported, initGrammars, loadGrammarsForLanguages } from './grammars';
|
||||
import { detectLanguage, isLanguageSupported, initGrammars, loadGrammarsForLanguages, resetParser } from './grammars';
|
||||
import { logDebug, logWarn } from '../errors';
|
||||
import { validatePathWithinRoot, normalizePath } from '../utils';
|
||||
import picomatch from 'picomatch';
|
||||
@@ -29,6 +29,12 @@ import picomatch from 'picomatch';
|
||||
*/
|
||||
const FILE_IO_BATCH_SIZE = 10;
|
||||
|
||||
/**
|
||||
* Reset tree-sitter parser after this many parses per language to reclaim
|
||||
* WASM heap memory and prevent "memory access out of bounds" crashes.
|
||||
*/
|
||||
const PARSER_RESET_INTERVAL = 5000;
|
||||
|
||||
/**
|
||||
* Progress callback for indexing operations
|
||||
*/
|
||||
@@ -412,6 +418,7 @@ export class ExtractionOrchestrator {
|
||||
// Phase 2: Parse files (read in parallel batches, parse/store sequentially)
|
||||
const total = files.length;
|
||||
let processed = 0;
|
||||
const parseCounts = new Map<Language, number>(); // track parses per language for WASM reset
|
||||
|
||||
for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
|
||||
if (signal?.aborted) {
|
||||
@@ -483,6 +490,16 @@ export class ExtractionOrchestrator {
|
||||
|
||||
const result = await this.indexFileWithContent(filePath, content, stats);
|
||||
|
||||
// Periodically reset the parser to reclaim WASM heap memory.
|
||||
// Without this, tree-sitter's WASM runtime fragments its heap
|
||||
// across thousands of parses and eventually crashes.
|
||||
const lang = detectLanguage(filePath);
|
||||
const count = (parseCounts.get(lang) ?? 0) + 1;
|
||||
parseCounts.set(lang, count);
|
||||
if (count % PARSER_RESET_INTERVAL === 0) {
|
||||
resetParser(lang);
|
||||
}
|
||||
|
||||
if (result.errors.length > 0) {
|
||||
// Annotate errors with file path if not already set
|
||||
for (const err of result.errors) {
|
||||
|
||||
Reference in New Issue
Block a user