Init
This commit is contained in:
Colby McHenry
+302
@@ -0,0 +1,302 @@
|
||||
/**
|
||||
* CodeGraph Utilities
|
||||
*
|
||||
* Common utility functions for memory management, concurrency, and batching.
|
||||
*
|
||||
* @module utils
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { Mutex, processInBatches, MemoryMonitor } from 'codegraph';
|
||||
*
|
||||
* // Use mutex for concurrent safety
|
||||
* const mutex = new Mutex();
|
||||
* await mutex.withLock(async () => {
|
||||
* await performCriticalOperation();
|
||||
* });
|
||||
*
|
||||
* // Process items in batches to manage memory
|
||||
* const results = await processInBatches(items, 100, async (item) => {
|
||||
* return await processItem(item);
|
||||
* });
|
||||
*
|
||||
* // Monitor memory usage
|
||||
* const monitor = new MemoryMonitor(512, (usage) => {
|
||||
* console.warn(`Memory usage exceeded 512MB: ${usage / 1024 / 1024}MB`);
|
||||
* });
|
||||
* monitor.start();
|
||||
* ```
|
||||
*/
|
||||
|
||||
/**
|
||||
* Process items in batches to manage memory
|
||||
*
|
||||
* @param items - Array of items to process
|
||||
* @param batchSize - Number of items per batch
|
||||
* @param processor - Function to process each item
|
||||
* @param onBatchComplete - Optional callback after each batch
|
||||
* @returns Array of results
|
||||
*/
|
||||
export async function processInBatches<T, R>(
|
||||
items: T[],
|
||||
batchSize: number,
|
||||
processor: (item: T, index: number) => Promise<R>,
|
||||
onBatchComplete?: (completed: number, total: number) => void
|
||||
): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
|
||||
for (let i = 0; i < items.length; i += batchSize) {
|
||||
const batch = items.slice(i, Math.min(i + batchSize, items.length));
|
||||
const batchResults = await Promise.all(
|
||||
batch.map((item, idx) => processor(item, i + idx))
|
||||
);
|
||||
results.push(...batchResults);
|
||||
|
||||
if (onBatchComplete) {
|
||||
onBatchComplete(Math.min(i + batchSize, items.length), items.length);
|
||||
}
|
||||
|
||||
// Allow GC between batches
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple mutex lock for preventing concurrent operations
|
||||
*/
|
||||
export class Mutex {
|
||||
private locked = false;
|
||||
private waitQueue: Array<() => void> = [];
|
||||
|
||||
/**
|
||||
* Acquire the lock
|
||||
*
|
||||
* @returns A release function to call when done
|
||||
*/
|
||||
async acquire(): Promise<() => void> {
|
||||
while (this.locked) {
|
||||
await new Promise<void>((resolve) => {
|
||||
this.waitQueue.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
this.locked = true;
|
||||
|
||||
return () => {
|
||||
this.locked = false;
|
||||
const next = this.waitQueue.shift();
|
||||
if (next) {
|
||||
next();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a function while holding the lock
|
||||
*/
|
||||
async withLock<T>(fn: () => Promise<T> | T): Promise<T> {
|
||||
const release = await this.acquire();
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the lock is currently held
|
||||
*/
|
||||
isLocked(): boolean {
|
||||
return this.locked;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunked file reader for large files
|
||||
*
|
||||
* Reads a file in chunks to avoid loading entire file into memory.
|
||||
*/
|
||||
export async function* readFileInChunks(
|
||||
filePath: string,
|
||||
chunkSize: number = 64 * 1024
|
||||
): AsyncGenerator<string, void, undefined> {
|
||||
const fs = await import('fs');
|
||||
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
const buffer = Buffer.alloc(chunkSize);
|
||||
|
||||
try {
|
||||
let bytesRead: number;
|
||||
while ((bytesRead = fs.readSync(fd, buffer, 0, chunkSize, null)) > 0) {
|
||||
yield buffer.toString('utf-8', 0, bytesRead);
|
||||
}
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Debounce a function
|
||||
*
|
||||
* @param fn - Function to debounce
|
||||
* @param delay - Delay in milliseconds
|
||||
* @returns Debounced function
|
||||
*/
|
||||
export function debounce<T extends (...args: unknown[]) => unknown>(
|
||||
fn: T,
|
||||
delay: number
|
||||
): (...args: Parameters<T>) => void {
|
||||
let timeoutId: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
return (...args: Parameters<T>) => {
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
timeoutId = setTimeout(() => {
|
||||
fn(...args);
|
||||
timeoutId = null;
|
||||
}, delay);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Throttle a function
|
||||
*
|
||||
* @param fn - Function to throttle
|
||||
* @param limit - Minimum time between calls in milliseconds
|
||||
* @returns Throttled function
|
||||
*/
|
||||
export function throttle<T extends (...args: unknown[]) => unknown>(
|
||||
fn: T,
|
||||
limit: number
|
||||
): (...args: Parameters<T>) => void {
|
||||
let lastCall = 0;
|
||||
let timeoutId: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
return (...args: Parameters<T>) => {
|
||||
const now = Date.now();
|
||||
const remaining = limit - (now - lastCall);
|
||||
|
||||
if (remaining <= 0) {
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
timeoutId = null;
|
||||
}
|
||||
lastCall = now;
|
||||
fn(...args);
|
||||
} else if (!timeoutId) {
|
||||
timeoutId = setTimeout(() => {
|
||||
lastCall = Date.now();
|
||||
timeoutId = null;
|
||||
fn(...args);
|
||||
}, remaining);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate memory usage of an object (rough approximation)
|
||||
*
|
||||
* @param obj - Object to measure
|
||||
* @returns Approximate size in bytes
|
||||
*/
|
||||
export function estimateSize(obj: unknown): number {
|
||||
const seen = new WeakSet();
|
||||
|
||||
function sizeOf(value: unknown): number {
|
||||
if (value === null || value === undefined) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
switch (typeof value) {
|
||||
case 'boolean':
|
||||
return 4;
|
||||
case 'number':
|
||||
return 8;
|
||||
case 'string':
|
||||
return 2 * (value as string).length;
|
||||
case 'object':
|
||||
if (seen.has(value as object)) {
|
||||
return 0;
|
||||
}
|
||||
seen.add(value as object);
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return value.reduce((acc: number, item) => acc + sizeOf(item), 0);
|
||||
}
|
||||
|
||||
return Object.entries(value as object).reduce(
|
||||
(acc, [key, val]) => acc + sizeOf(key) + sizeOf(val),
|
||||
0
|
||||
);
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return sizeOf(obj);
|
||||
}
|
||||
|
||||
/**
|
||||
* Memory monitor for tracking usage during operations
|
||||
*/
|
||||
export class MemoryMonitor {
|
||||
private checkInterval: ReturnType<typeof setInterval> | null = null;
|
||||
private peakUsage = 0;
|
||||
private threshold: number;
|
||||
private onThresholdExceeded?: (usage: number) => void;
|
||||
|
||||
constructor(
|
||||
thresholdMB: number = 500,
|
||||
onThresholdExceeded?: (usage: number) => void
|
||||
) {
|
||||
this.threshold = thresholdMB * 1024 * 1024;
|
||||
this.onThresholdExceeded = onThresholdExceeded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start monitoring memory usage
|
||||
*/
|
||||
start(intervalMs: number = 1000): void {
|
||||
this.stop();
|
||||
this.peakUsage = 0;
|
||||
|
||||
this.checkInterval = setInterval(() => {
|
||||
const usage = process.memoryUsage().heapUsed;
|
||||
if (usage > this.peakUsage) {
|
||||
this.peakUsage = usage;
|
||||
}
|
||||
if (usage > this.threshold && this.onThresholdExceeded) {
|
||||
this.onThresholdExceeded(usage);
|
||||
}
|
||||
}, intervalMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop monitoring
|
||||
*/
|
||||
stop(): void {
|
||||
if (this.checkInterval) {
|
||||
clearInterval(this.checkInterval);
|
||||
this.checkInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get peak memory usage in bytes
|
||||
*/
|
||||
getPeakUsage(): number {
|
||||
return this.peakUsage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current memory usage in bytes
|
||||
*/
|
||||
getCurrentUsage(): number {
|
||||
return process.memoryUsage().heapUsed;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user