perf(kernel): direct-to-store decode — buffers flow to the store worker, main thread never materializes nodes
Kernel-routed files ship their flat tables from the parse worker to the store worker as buffers (tryKernelExtractRaw → kernelBuffers on the result → KernelStoreBundle); the store worker decodes and finalizes (finalizeStoreBundle shared with the object path so filter semantics can never drift). Files with applicable framework extract() hooks keep the decoded path; the no-writer fallback materializes via materializeKernelResult. Byte-identical dumps re-verified on dubbo, excalidraw, express, gson; full suite green (2,467). Measurement (plan §4d): dubbo's parse-loop wall is 94% store-writer busy time — the many-core fresh-index wall is single-writer SQLite ingest, not extraction or main-thread work. d2s improves the writer lane ~11% (structured-clone deserialization avoided on the writer) and frees the main thread; the remaining many-core gap is a store-architecture arc (deferred index builds, multi-file transactions, buffer→bind), out of the kernel project's scope. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
03d54e47a1
commit
28068fa0f1
@@ -16,6 +16,8 @@ try {
|
||||
import { parentPort } from 'worker_threads';
|
||||
import { extractFromSource } from './tree-sitter';
|
||||
import { detectLanguage, loadGrammarsForLanguages, resetParser } from './grammars';
|
||||
import { tryKernelExtractRaw } from './kernel';
|
||||
import { getAllFrameworkResolvers, getApplicableFrameworks } from '../resolution/frameworks';
|
||||
import type { Language, ExtractionResult } from '../types';
|
||||
|
||||
// Emscripten prints `Aborted()` (and a follow-up RuntimeError diag
|
||||
@@ -80,7 +82,36 @@ parentPort!.on('message', async (msg: { type: string; id?: number; filePath?: st
|
||||
// codegraph.json extension overrides) and sends it; fall back to detection
|
||||
// for older callers / safety.
|
||||
const language = msg.language ?? detectLanguage(filePath!, content);
|
||||
const result: ExtractionResult = extractFromSource(filePath!, content!, language, frameworkNames);
|
||||
|
||||
// Kernel deferred-decode fast path: ship the file's tables as flat
|
||||
// buffers and decode at the STORE boundary, so the main thread never
|
||||
// materializes per-node objects (nor pays their structured-clone cost —
|
||||
// buffer clone is a flat memcpy). Only when no applicable framework has
|
||||
// an extract() hook: those merge extra nodes/refs into the DECODED
|
||||
// result inside extractFromSource, so such files keep the decoded path.
|
||||
let result: ExtractionResult | undefined;
|
||||
const frameworksNeedDecode =
|
||||
frameworkNames && frameworkNames.length > 0
|
||||
? getApplicableFrameworks(
|
||||
getAllFrameworkResolvers().filter((r) => frameworkNames.includes(r.name)),
|
||||
language
|
||||
).some((fw) => !!fw.extract)
|
||||
: false;
|
||||
if (!frameworksNeedDecode) {
|
||||
const raw = tryKernelExtractRaw(filePath!, content!, language);
|
||||
if (raw) {
|
||||
result = {
|
||||
nodes: [],
|
||||
edges: [],
|
||||
unresolvedReferences: [],
|
||||
errors: raw.errors,
|
||||
durationMs: 0,
|
||||
kernelBuffers: raw.buffers,
|
||||
kernelCounts: raw.counts,
|
||||
};
|
||||
}
|
||||
}
|
||||
result ??= extractFromSource(filePath!, content!, language, frameworkNames);
|
||||
|
||||
// Periodic parser reset to reclaim WASM heap memory
|
||||
const count = (parseCounts.get(language) ?? 0) + 1;
|
||||
|
||||
Reference in New Issue
Block a user