perf(kernel): cFnPtr native extraction sweep — step 2, pass 230→151s across the arc (§7a.10) (#1365)
Task #5 step 2. The fuse-then-link refactor (#1364) left the extraction sweep as a clean per-file boundary: raw text in → collected facts out. This ports that sweep to the native kernel: `cfnptr_scan_files` (codegraph-kernel/src/cfnptr.rs) strips and scans a batch of 16 files per NAPI call, and the TS side only reads files, ships batches, interns the returned facts, and resolves include paths. The JS sweep remains as the fallback (no binary, feature detection against older binaries, CODEGRAPH_KERNEL=0, or CODEGRAPH_KERNEL_CFNPTR=0). Parity discipline: the JS regexes are the spec, so the scanners are hand-rolled byte machines reproducing that engine — ASCII \w/\b next to UNICODE \s (NBSP/U+2000-200A/FEFF decoded from UTF-8), alternation order, lastIndex resume, and the observable backtracking dimensions (INIT/ARRAY modifier and struct/star/bracket optionals, DISPATCH's greedy segment loop); greedy shortcuts only where backtracking provably can't rescue a match. The native stripper blanks per UTF-16 code unit, so its output is string-identical to the TS stripper — pinned by a new kernel arm on the strip differential oracle (fixtures + 500 seeded random cases). Gates, all green: new differential suite (adversarial fixture project — CRLF, NBSP, continuations, decoy strings, unterminated comments, backtracking shapes — indexed native-vs-JS: identical edge streams, plus a record-level scanner check); repo differential on git/redis/vim/SameBoy (identical, 705/852/433/180 edges); probe-hash on the live linux kernel DB reproduced f6e1713d… (279,335 rows); linux init counts exact 2,049,153/6,413,518; dump sha 6dd1185b… reproduced (10,446,478 lines); full suite green ×2 (153 files / 2588 tests). Measured (8c cg1212, quiet host): cFnPtr sub A=47.9s B=1.1 C=40.9 D=24.1 E=36.8 = 150.9s vs step 1's 179s and the pre-arc 230s (−34% cumulative); the sweep itself halved (94.5→47.9s, JS strips 132.4k→68.9k). callback-synthesis phase 199.9→171.1s. E's attributed wall grew from overlap shift under parallel synthesis; the phase total is the honest number. Full record: plan §7a.10. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
c6850d737b
commit
69ea438bac
@@ -53,10 +53,41 @@ export interface KernelGrammarInfo {
|
||||
fieldNames: string[];
|
||||
}
|
||||
|
||||
/** Input to the cFnPtr extraction sweep: one file's raw text + its struct
|
||||
* node extents (`endLine ?? startLine` applied by the caller). */
|
||||
export interface CfnptrFileIn {
|
||||
text: string;
|
||||
structs: { id: string; startLine: number; endLine: number }[];
|
||||
}
|
||||
|
||||
/** Per-file facts from the native cFnPtr extraction sweep — mirror of the
|
||||
* Rust `CfnptrFacts` (see codegraph-kernel/src/cfnptr.rs); semantics match
|
||||
* the JS sweep in src/resolution/c-fnptr-synthesizer.ts. */
|
||||
export interface CfnptrFactsOut {
|
||||
fnPtrTypedefs: string[];
|
||||
fnTypeTypedefs: string[];
|
||||
structs: { id: string; parsed: boolean; fields: { name: string; index: number; ptr: boolean; type: string }[] }[];
|
||||
inlinePtr: boolean;
|
||||
inlineTypes: string[];
|
||||
inlineTags: string[];
|
||||
initTokens: string[];
|
||||
arrayElems: string[];
|
||||
aliasNames: string[];
|
||||
dPairs: string[];
|
||||
dispatchFields: string[];
|
||||
arrayDispatchNames: string[];
|
||||
includes: string[];
|
||||
}
|
||||
|
||||
export interface KernelModule {
|
||||
extractFile(filePath: string, content: string, language: string): KernelBuffers;
|
||||
contractInfo(): KernelContractInfo;
|
||||
grammarInfo(language: string): KernelGrammarInfo | null;
|
||||
/** Batched cFnPtr extraction sweep (task #5 step 2). OPTIONAL: absent on
|
||||
* older binaries — callers feature-detect and keep their JS path. */
|
||||
cfnptrScanFiles?(files: CfnptrFileIn[]): CfnptrFactsOut[];
|
||||
/** Native `stripCommentsForRegex(text, 'c')` — differential-oracle hook. */
|
||||
cfnptrStripC?(text: string): string;
|
||||
}
|
||||
|
||||
const debugEnabled = () => process.env.CODEGRAPH_KERNEL_DEBUG === '1';
|
||||
|
||||
@@ -88,6 +88,8 @@ import type { MaybeYield } from './cooperative-yield';
|
||||
import { memoryBudgetBytes } from './memory-budget';
|
||||
import { LRUCache } from './lru-cache';
|
||||
import { stripCommentsForRegex } from './strip-comments';
|
||||
import { getKernel } from '../extraction/kernel/loader';
|
||||
import type { CfnptrFactsOut, CfnptrFileIn } from '../extraction/kernel/loader';
|
||||
|
||||
const C_CPP_EXT = /\.(c|h|cc|cpp|cxx|hpp|hh|hxx|cppm|ipp|inl|tcc)$/i;
|
||||
const FN_KINDS = new Set(['function', 'method']);
|
||||
@@ -627,8 +629,93 @@ export async function cFnPointerDispatchEdges(
|
||||
};
|
||||
|
||||
// ---- Stage A: the extraction sweep — ONE read + strip per file ----
|
||||
//
|
||||
// Two implementations, record-identical by the differential suite:
|
||||
// • native (task #5 step 2): the kernel's `cfnptrScanFiles` strips and
|
||||
// scans a BATCH of files per NAPI call (codegraph-kernel/src/cfnptr.rs —
|
||||
// hand-rolled byte machines replicating the JS regex semantics), and the
|
||||
// TS side only reads files, ships batches, and interns the returned
|
||||
// facts. Include-path resolution stays here (it needs the filesystem).
|
||||
// • JS: the original sweep, kept verbatim — the fallback for platforms
|
||||
// without a kernel binary, older binaries (feature detection), the
|
||||
// CODEGRAPH_KERNEL=0 kill switch, and CODEGRAPH_KERNEL_CFNPTR=0 (this
|
||||
// scanner's own switch).
|
||||
const kernel =
|
||||
process.env.CODEGRAPH_KERNEL === '0' || process.env.CODEGRAPH_KERNEL_CFNPTR === '0'
|
||||
? null
|
||||
: getKernel();
|
||||
const nativeSweep =
|
||||
kernel && typeof kernel.cfnptrScanFiles === 'function' ? kernel.cfnptrScanFiles.bind(kernel) : null;
|
||||
|
||||
const mergeNativeFacts = (file: string, out: CfnptrFactsOut): void => {
|
||||
for (const t of out.fnPtrTypedefs) fnPtrTypedefs.add(intern(t));
|
||||
for (const t of out.fnTypeTypedefs) fnTypeTypedefs.add(intern(t));
|
||||
for (const so of out.structs) {
|
||||
if (!so.parsed) continue; // body never parsed — the JS sweep records nothing either
|
||||
rawFieldsByNode.set(
|
||||
so.id,
|
||||
so.fields.map((f) => ({ name: f.name || null, index: f.index, ptr: f.ptr, type: f.type }))
|
||||
);
|
||||
}
|
||||
for (const t of out.inlineTags) inlineTags.add(intern(t));
|
||||
for (const t of out.aliasNames) aliasNames.add(intern(t));
|
||||
const includes: string[] = [];
|
||||
for (const cap of out.includes) {
|
||||
if (!INCLUDABLE_EXT.test(cap)) continue;
|
||||
const t = resolveInclude(file, cap);
|
||||
if (t) includes.push(intern(t));
|
||||
}
|
||||
if (
|
||||
out.initTokens.length || out.arrayElems.length || out.inlinePtr || out.inlineTypes.length ||
|
||||
out.dPairs.length || out.dispatchFields.length || out.arrayDispatchNames.length || includes.length
|
||||
) {
|
||||
factsByFile.set(file, {
|
||||
initTokens: out.initTokens.length ? out.initTokens.map(intern) : null,
|
||||
arrayElems: out.arrayElems.length ? out.arrayElems.map(intern) : null,
|
||||
inlinePtr: out.inlinePtr,
|
||||
inlineTypes: out.inlineTypes.length ? out.inlineTypes.map(intern) : null,
|
||||
dPairs: out.dPairs.length ? out.dPairs.map(intern) : null,
|
||||
dispatchFields: out.dispatchFields.length ? out.dispatchFields.map(intern) : null,
|
||||
arrayDispatchNames: out.arrayDispatchNames.length ? out.arrayDispatchNames.map(intern) : null,
|
||||
includes: includes.length ? includes : NO_INCLUDES,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
let tPass = Date.now();
|
||||
for (const file of files) {
|
||||
if (nativeSweep) {
|
||||
// Batch of 16 = the tick/onFraction cadence, so yielding and progress
|
||||
// reporting keep their shape while the boundary crossing amortizes.
|
||||
const BATCH = 16;
|
||||
let batch: { file: string; input: CfnptrFileIn }[] = [];
|
||||
const flush = (): void => {
|
||||
if (batch.length === 0) return;
|
||||
const outs = nativeSweep(batch.map((b) => b.input));
|
||||
for (let bi = 0; bi < batch.length; bi++) mergeNativeFacts(batch[bi]!.file, outs[bi]!);
|
||||
batch = [];
|
||||
};
|
||||
for (const file of files) {
|
||||
await tick();
|
||||
const rawText = raw(file);
|
||||
if (!rawText) continue; // unreadable or empty — the JS sweep skips these too
|
||||
const tN = prof ? Date.now() : 0;
|
||||
const fileNodes = ctx.getNodesInFile(file);
|
||||
if (prof) { prof.nodesMs += Date.now() - tN; prof.nodesN++; }
|
||||
const structs: CfnptrFileIn['structs'] = [];
|
||||
for (const st of fileNodes) {
|
||||
if (st.kind !== 'struct') continue;
|
||||
// sliceLinesPre semantics ride along: falsy startLine never parses,
|
||||
// and `endLine ?? startLine` is applied here so the kernel sees the
|
||||
// exact slice bounds the JS sweep would use.
|
||||
structs.push({ id: st.id, startLine: st.startLine ?? 0, endLine: st.endLine ?? st.startLine ?? 0 });
|
||||
}
|
||||
batch.push({ file, input: { text: rawText, structs } });
|
||||
if (batch.length >= BATCH) flush();
|
||||
}
|
||||
flush();
|
||||
}
|
||||
// JS sweep (fallback path — see the stage comment above).
|
||||
if (!nativeSweep) for (const file of files) {
|
||||
await tick();
|
||||
const s = src(file);
|
||||
if (!s) continue;
|
||||
|
||||
Reference in New Issue
Block a user