feat(kernel): R7a C/C++ walker — dual-lang ccpp module, preParse hoist, 7 new blanks, c/cpp default-routed (#1346)

Parity: 0 diffs on redis/git/fmt/protobuf/ALS sweeps; full-init dumps
byte-identical on all five + linux at kernel scale (10.4M dump lines,
same sha256 both arms). Linux 2c/6GB envelope: kernel-arm 19.1min vs
wasm-arm 22.9min (parse 356s vs 435s) on a much richer graph (the new
blanks recover error-swallowed code: git 2x nodes, linux kernel/+mm/ 3x).
Deferral guard corrected by measurement (C/C++ error incidence 9-42%;
--max-deferral flag); defer-reuse memo kills the 3x re-blank/re-parse
cost deferred files paid.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
2026-07-17 16:56:41 -05:00
committed by GitHub
co-authored by Claude Fable 5
parent 44561b6aad
commit 2d72891b59
20 changed files with 3211 additions and 80 deletions
+68 -8
View File
@@ -15,6 +15,7 @@
*/
import type { ExtractionResult, Language } from '../../types';
import { EXTRACTORS } from '../languages';
import { getKernel, kernelSupports } from './loader';
import { decodeExtractBuffers } from './decode';
import {
@@ -41,6 +42,12 @@ const DEFAULT_ROUTED: ReadonlySet<Language> = new Set<Language>([
'java',
'python',
'go',
// R7a (2026-07-17): parity swept 0-diff on redis/git/fmt/protobuf/ALS
// (2,389 files compared) + full-init dump-diffs byte-identical; erroring
// files defer per-file to wasm (routine for macro-heavy C/C++ — see
// scripts/kernel-parity.mjs --max-deferral).
'c',
'cpp',
]);
/**
@@ -55,6 +62,22 @@ const POST_PASSES: Partial<Record<Language, KernelPostPass>> = {
// (none yet — R2+)
};
/**
* The preParse hoist (checklist §arch-1): languages with an offset-preserving
* `preParse` hook (c/cpp macro blanking, csharp #237, metal #1121, cuda #1172)
* apply it HERE, before the kernel call, so both arms parse identical blanked
* bytes and none of the blanking logic needs a Rust port. The wasm fallback
* path is untouched — TreeSitterExtractor applies the same hook itself on the
* RAW source it receives, so a kernel error/defer still extracts identically.
* Every blank is an equal-length-space replacement, so offsets, lines, and
* columns survive; `filePath` rides along for the extension-gated dialect
* blanks (`.metal` attributes; `.cu`/`.cuh` + content-gated CUDA).
*/
function preParsedSource(filePath: string, source: string, language: Language): string {
const pre = EXTRACTORS[language]?.preParse;
return pre ? pre(source, filePath) : source;
}
function isRouted(language: Language): boolean {
const env = process.env.CODEGRAPH_KERNEL_LANGS;
if (env === undefined || env === '') return DEFAULT_ROUTED.has(language);
@@ -73,6 +96,37 @@ export function kernelRoutes(language: Language): boolean {
/** Warned-once registry so a broken language logs a single line, not one per file. */
const warned = new Set<string>();
/**
* One-slot defer memo. A file the kernel defers (parse errors → wasm) used to
* pay the full pipeline again at every seam: the worker's raw try blanked +
* native-parsed it, extractFromSource's kernel try blanked + native-parsed it
* AGAIN, and the wasm extractor then re-applied preParse a third time. On a
* high-deferral tree (the Linux kernel defers ~79% of files) that waste
* dominated the arm's parse phase. The slot remembers the LAST deferred
* (file, source, language) so (a) a repeat kernel attempt for the same file
* short-circuits to null, and (b) the wasm fallback can reuse the
* already-blanked source instead of re-running preParse. Source is matched by
* string identity — the worker passes the same string through every seam.
*/
let deferSlot: { filePath: string; source: string; language: Language; pre: string } | null = null;
/** The hoisted preParse output for a just-deferred file, if it matches. */
export function takeDeferredPreParse(
filePath: string,
source: string,
language: Language
): string | null {
if (
deferSlot &&
deferSlot.filePath === filePath &&
deferSlot.source === source &&
deferSlot.language === language
) {
return deferSlot.pre;
}
return null;
}
/** The raw table buffers + the cheap facts the orchestrator needs pre-decode. */
export interface KernelRawResult {
buffers: NonNullable<ExtractionResult['kernelBuffers']>;
@@ -96,8 +150,10 @@ export function tryKernelExtractRaw(
if (!kernelRoutes(language) || POST_PASSES[language]) return null;
const kernel = getKernel();
if (!kernel) return null;
if (takeDeferredPreParse(filePath, source, language) !== null) return null; // already deferred
const pre = preParsedSource(filePath, source, language);
try {
const buffers = kernel.extractFile(filePath, source, language);
const buffers = kernel.extractFile(filePath, pre, language);
const meta = buffers.meta;
if (meta.readUInt8(LAYOUT_META.version) !== LAYOUT_ABI) {
throw new Error(`kernel buffer ABI ${meta.readUInt8(0)} != expected ${LAYOUT_ABI}`);
@@ -118,7 +174,10 @@ export function tryKernelExtractRaw(
return { buffers, counts, errors };
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
if (message.includes('defer:')) return null;
if (message.includes('defer:')) {
deferSlot = { filePath, source, language, pre };
return null;
}
if (!warned.has(language)) {
warned.add(language);
process.stderr.write(
@@ -166,13 +225,11 @@ export function tryKernelExtract(
if (!kernelRoutes(language)) return null;
const kernel = getKernel();
if (!kernel) return null;
if (takeDeferredPreParse(filePath, source, language) !== null) return null; // already deferred
const t0 = Date.now();
const pre = preParsedSource(filePath, source, language);
try {
// NOTE(T2 languages): when a preParse-carrying language (csharp #237,
// metal #1121, cuda #1172, c/cpp macro blanking) routes here, its
// offset-preserving preParse hook must be applied to `source` first —
// wire that alongside the language's port, gated WITH its equivalence run.
const buffers = kernel.extractFile(filePath, source, language);
const buffers = kernel.extractFile(filePath, pre, language);
const result = decodeExtractBuffers(buffers, filePath, language);
POST_PASSES[language]?.(result, source);
result.durationMs = Date.now() - t0;
@@ -182,7 +239,10 @@ export function tryKernelExtract(
// `defer:` is the kernel's expected-routing signal (files with parse
// errors take the wasm path — its error RECOVERY is the canonical one;
// recovery differs between UTF-8 and UTF-16 parsing). Silent by design.
if (message.includes('defer:')) return null;
if (message.includes('defer:')) {
deferSlot = { filePath, source, language, pre };
return null;
}
if (!warned.has(language)) {
warned.add(language);
process.stderr.write(