feat(kernel): R7a C/C++ walker — dual-lang ccpp module, preParse hoist, 7 new blanks, c/cpp default-routed (#1346)

Parity: 0 diffs on redis/git/fmt/protobuf/ALS sweeps; full-init dumps
byte-identical on all five + linux at kernel scale (10.4M dump lines,
same sha256 both arms). Linux 2c/6GB envelope: kernel-arm 19.1min vs
wasm-arm 22.9min (parse 356s vs 435s) on a much richer graph (the new
blanks recover error-swallowed code: git 2x nodes, linux kernel/+mm/ 3x).
Deferral guard corrected by measurement (C/C++ error incidence 9-42%;
--max-deferral flag); defer-reuse memo kills the 3x re-blank/re-parse
cost deferred files paid.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
2026-07-17 16:56:41 -05:00
committed by GitHub
co-authored by Claude Fable 5
parent 44561b6aad
commit 2d72891b59
20 changed files with 3211 additions and 80 deletions
+68 -8
View File
@@ -15,6 +15,7 @@
*/
import type { ExtractionResult, Language } from '../../types';
import { EXTRACTORS } from '../languages';
import { getKernel, kernelSupports } from './loader';
import { decodeExtractBuffers } from './decode';
import {
@@ -41,6 +42,12 @@ const DEFAULT_ROUTED: ReadonlySet<Language> = new Set<Language>([
'java',
'python',
'go',
// R7a (2026-07-17): parity swept 0-diff on redis/git/fmt/protobuf/ALS
// (2,389 files compared) + full-init dump-diffs byte-identical; erroring
// files defer per-file to wasm (routine for macro-heavy C/C++ — see
// scripts/kernel-parity.mjs --max-deferral).
'c',
'cpp',
]);
/**
@@ -55,6 +62,22 @@ const POST_PASSES: Partial<Record<Language, KernelPostPass>> = {
// (none yet — R2+)
};
/**
* The preParse hoist (checklist §arch-1): languages with an offset-preserving
* `preParse` hook (c/cpp macro blanking, csharp #237, metal #1121, cuda #1172)
* apply it HERE, before the kernel call, so both arms parse identical blanked
* bytes and none of the blanking logic needs a Rust port. The wasm fallback
* path is untouched — TreeSitterExtractor applies the same hook itself on the
* RAW source it receives, so a kernel error/defer still extracts identically.
* Every blank is an equal-length-space replacement, so offsets, lines, and
* columns survive; `filePath` rides along for the extension-gated dialect
* blanks (`.metal` attributes; `.cu`/`.cuh` + content-gated CUDA).
*/
function preParsedSource(filePath: string, source: string, language: Language): string {
const pre = EXTRACTORS[language]?.preParse;
return pre ? pre(source, filePath) : source;
}
function isRouted(language: Language): boolean {
const env = process.env.CODEGRAPH_KERNEL_LANGS;
if (env === undefined || env === '') return DEFAULT_ROUTED.has(language);
@@ -73,6 +96,37 @@ export function kernelRoutes(language: Language): boolean {
/** Warned-once registry so a broken language logs a single line, not one per file. */
const warned = new Set<string>();
/**
* One-slot defer memo. A file the kernel defers (parse errors → wasm) used to
* pay the full pipeline again at every seam: the worker's raw try blanked +
* native-parsed it, extractFromSource's kernel try blanked + native-parsed it
* AGAIN, and the wasm extractor then re-applied preParse a third time. On a
* high-deferral tree (the Linux kernel defers ~79% of files) that waste
* dominated the arm's parse phase. The slot remembers the LAST deferred
* (file, source, language) so (a) a repeat kernel attempt for the same file
* short-circuits to null, and (b) the wasm fallback can reuse the
* already-blanked source instead of re-running preParse. Source is matched by
* string identity — the worker passes the same string through every seam.
*/
let deferSlot: { filePath: string; source: string; language: Language; pre: string } | null = null;
/** The hoisted preParse output for a just-deferred file, if it matches. */
export function takeDeferredPreParse(
filePath: string,
source: string,
language: Language
): string | null {
if (
deferSlot &&
deferSlot.filePath === filePath &&
deferSlot.source === source &&
deferSlot.language === language
) {
return deferSlot.pre;
}
return null;
}
/** The raw table buffers + the cheap facts the orchestrator needs pre-decode. */
export interface KernelRawResult {
buffers: NonNullable<ExtractionResult['kernelBuffers']>;
@@ -96,8 +150,10 @@ export function tryKernelExtractRaw(
if (!kernelRoutes(language) || POST_PASSES[language]) return null;
const kernel = getKernel();
if (!kernel) return null;
if (takeDeferredPreParse(filePath, source, language) !== null) return null; // already deferred
const pre = preParsedSource(filePath, source, language);
try {
const buffers = kernel.extractFile(filePath, source, language);
const buffers = kernel.extractFile(filePath, pre, language);
const meta = buffers.meta;
if (meta.readUInt8(LAYOUT_META.version) !== LAYOUT_ABI) {
throw new Error(`kernel buffer ABI ${meta.readUInt8(0)} != expected ${LAYOUT_ABI}`);
@@ -118,7 +174,10 @@ export function tryKernelExtractRaw(
return { buffers, counts, errors };
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
if (message.includes('defer:')) return null;
if (message.includes('defer:')) {
deferSlot = { filePath, source, language, pre };
return null;
}
if (!warned.has(language)) {
warned.add(language);
process.stderr.write(
@@ -166,13 +225,11 @@ export function tryKernelExtract(
if (!kernelRoutes(language)) return null;
const kernel = getKernel();
if (!kernel) return null;
if (takeDeferredPreParse(filePath, source, language) !== null) return null; // already deferred
const t0 = Date.now();
const pre = preParsedSource(filePath, source, language);
try {
// NOTE(T2 languages): when a preParse-carrying language (csharp #237,
// metal #1121, cuda #1172, c/cpp macro blanking) routes here, its
// offset-preserving preParse hook must be applied to `source` first —
// wire that alongside the language's port, gated WITH its equivalence run.
const buffers = kernel.extractFile(filePath, source, language);
const buffers = kernel.extractFile(filePath, pre, language);
const result = decodeExtractBuffers(buffers, filePath, language);
POST_PASSES[language]?.(result, source);
result.durationMs = Date.now() - t0;
@@ -182,7 +239,10 @@ export function tryKernelExtract(
// `defer:` is the kernel's expected-routing signal (files with parse
// errors take the wasm path — its error RECOVERY is the canonical one;
// recovery differs between UTF-8 and UTF-16 parsing). Silent by design.
if (message.includes('defer:')) return null;
if (message.includes('defer:')) {
deferSlot = { filePath, source, language, pre };
return null;
}
if (!warned.has(language)) {
warned.add(language);
process.stderr.write(
+357 -14
View File
@@ -520,6 +520,63 @@ export function blankCppAnnotationMacroCalls(source: string): string {
return chars.join('');
}
/**
* Blank a macro that is the ONLY token on its line — no parens, no semicolon:
* namespace-management macros (`FMT_BEGIN_NAMESPACE`, `FMT_END_EXPORT`,
* `JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS`), Qt's `Q_OBJECT`, and friends. A
* bare identifier is not a statement or declaration in C or C++, so
* tree-sitter drops into error recovery at every one — and since the kernel
* path defers ANY erroring file to wasm, this single idiom deferred 13/73 fmt
* files and a comparable share of jemalloc, forfeiting the native-parse win
* on exactly the header-heavy trees it targets (the wasm path also mis-nests
* scopes around them today). Replacing the token with equal-length spaces
* preserves every byte offset and the surrounding declarations parse clean.
*
* Matched tightly so a real identifier can never be touched — ALL of:
* - the line consists of ONE ALL-CAPS token (≥4 chars, with `_`), optionally
* followed by a same-line comment — a lone lowercase identifier or any
* second token disqualifies;
* - the PREVIOUS non-blank line does not end in a continuation character
* (`=`, an operator, `,`, `(`, `?`, `:`, or a `\` macro-definition
* continuation) — so an ALL-CAPS operand split onto its own line inside a
* multi-line expression (`int x =\n SOME_CONST\n | OTHER;`) is left
* alone; and
* - the NEXT non-blank line starts like a declaration/scope token
* (letter, `_`, `#`, `{`, `}`, or `~`) or the file ends — an operator,
* string literal, or `;` continuation rejects the match.
* Shared by C and C++ (the idiom is identical in both).
*/
const LONE_MACRO_LINE_RE = /^[ \t]*([A-Z][A-Z0-9_]{3,})[ \t]*(?:\/\/[^\n\r]*|\/\*[^\n\r]*\*\/[ \t]*)?\r?$/;
const LONE_MACRO_CONTINUATION_END_RE = /[=+\-*/%&|^<>?:,(\\]$/;
export function blankLoneMacroLines(source: string): string {
if (!/^[ \t]*[A-Z][A-Z0-9_]{3,}[ \t]*\r?$/m.test(source)) return source;
const lines = source.split('\n');
const content = (l: string): string => l.replace(/\r$/, '').trim();
let changed = false;
for (let i = 0; i < lines.length; i++) {
const line = lines[i] as string;
const m = LONE_MACRO_LINE_RE.exec(line);
if (!m) continue;
// Underscore requirement rides the macro convention (FMT_BEGIN_NAMESPACE,
// Q_OBJECT); a solid all-caps word (`NDEBUG`-style) alone is too risky.
if (!(m[1] as string).includes('_')) continue;
let prev = i - 1;
while (prev >= 0 && content(lines[prev] as string) === '') prev--;
if (prev >= 0 && LONE_MACRO_CONTINUATION_END_RE.test(content(lines[prev] as string))) continue;
let next = i + 1;
while (next < lines.length && content(lines[next] as string) === '') next++;
if (next < lines.length) {
const first = content(lines[next] as string)[0];
if (!first || !/[A-Za-z_#{}~]/.test(first)) continue;
}
const start = line.indexOf(m[1] as string);
lines[i] =
line.slice(0, start) + ' '.repeat((m[1] as string).length) + line.slice(start + (m[1] as string).length);
changed = true;
}
return changed ? lines.join('\n') : source;
}
/**
* Blank an export/visibility macro sitting in front of a *member* or *method*
* declaration inside a class/namespace (`ENGINE_API virtual void Tick(…)`,
@@ -689,24 +746,64 @@ function looksLikeCudaSource(source: string): boolean {
);
}
/**
* Restore preprocessor-directive lines to their original bytes after the
* blanking passes ran. The token-level blanks match on shape, not context, so
* a macro name that happens to sit inside a DIRECTIVE gets blanked too — and
* blanking the name position of `#define FMT_API FMT_VISIBILITY("default")`
* leaves a nameless `# define FMT_VISIBILITY(…)`, which is a parse
* ERROR (fmt's base.h carries several). Inside a directive the blanks were
* never useful anyway: tree-sitter stores `#define` bodies as raw
* preproc_arg text it doesn't parse, so blanking there can only ever break
* the directive itself. Copying the original directive lines back (including
* `\`-continuation lines of multi-line defines) is offset-preserving by
* construction and strictly reduces parse errors on both extraction arms.
*/
function restoreDirectiveLines(original: string, blanked: string): string {
if (blanked === original || original.indexOf('#') === -1) return blanked;
const o = original.split('\n');
const b = blanked.split('\n');
let changed = false;
let continuation: boolean = false;
for (let i = 0; i < o.length && i < b.length; i++) {
const line = o[i] as string;
const isDirective: boolean = continuation || /^[ \t]*#/.test(line);
if (isDirective && b[i] !== line) {
b[i] = line;
changed = true;
}
continuation = isDirective && /\\\s*$/.test(line.replace(/\r$/, ''));
}
return changed ? b.join('\n') : blanked;
}
/** C/C++ source pre-processing before tree-sitter: recover macro-annotated class
* definitions, macro-prefixed function definitions, macro-prefixed members, and
* macro-decorated members (Unreal-Engine reflection markup) — plus the non-C++
* surface of the dialects parsed with the C++ grammar: `.metal` MSL attribute
* annotations, and CUDA specifiers + launch syntax (by `.cu`/`.cuh` extension
* or by content, for CUDA living in `.h`/`.hpp` headers). Offset-preserving. */
* or by content, for CUDA living in `.h`/`.hpp` headers). Offset-preserving;
* directive lines are restored at the end (see restoreDirectiveLines). */
function preParseCppSource(source: string, filePath?: string): string {
const blanked = blankCppAnnotationMacroCalls(
blankCppInlineAnnotationMacros(
blankCppApiPrefixMacros(blankCppInlineMacros(blankCppExportMacros(source)))
// blankCLeadingAttrMacros runs AFTER the api-prefix blank so a stacked
// `FMT_NORETURN FMT_API void f(…)` reduces to the `MACRO Ret name(` shape
// it matches (the _API token is already spaces by then).
let blanked = blankLoneMacroLines(
blankCLeadingAttrMacros(
blankCppAnnotationMacroCalls(
blankCppInlineAnnotationMacros(
blankCppApiPrefixMacros(blankCppInlineMacros(blankCppExportMacros(source)))
)
)
)
);
const lower = filePath ? filePath.toLowerCase() : '';
if (lower.endsWith('.metal')) return blankMetalAttributes(blanked);
if (lower.endsWith('.cu') || lower.endsWith('.cuh') || looksLikeCudaSource(source)) {
return blankCudaConstructs(blanked);
if (lower.endsWith('.metal')) {
blanked = blankMetalAttributes(blanked);
} else if (lower.endsWith('.cu') || lower.endsWith('.cuh') || looksLikeCudaSource(source)) {
blanked = blankCudaConstructs(blanked);
}
return blanked;
return restoreDirectiveLines(source, blanked);
}
/**
@@ -743,13 +840,259 @@ export function blankCLeadingAttrMacros(source: string): string {
);
}
/** C source pre-processing: recover functions hidden behind a leading
* attribute macro (#1211), then — for C-detected headers in CUDA projects
* (llm.c keeps `__device__` helpers and kernel prototypes in plain `.h`) —
* the same content-gated CUDA blank as C++. Offset-preserving. */
/**
* Blank the body of `#ifdef __cplusplus … #endif` guard regions in C sources.
* The ubiquitous C-header compatibility idiom
*
* #ifdef __cplusplus
* extern "C" {
* #endif
*
* is NOT valid C — `extern "C" {` (and any other C++-only line under the
* guard) drops tree-sitter-c into error recovery, so effectively every public
* C header carries parse errors. The wasm path shrugs (recovery keeps the
* rest); the kernel path defers EVERY erroring file to wasm by policy — so
* this one idiom pushed C-header deferral to ~32% on redis (vs the <10%
* gate) and forfeited the native-parse win exactly where C repos have the
* most files. A C compiler never sees the guarded lines (`__cplusplus` is
* only defined for C++), so blanking the region BODY mirrors the
* preprocessor's own view of the file.
*
* Matched conservatively, line-based and offset-preserving:
* - the opener must be `#ifdef __cplusplus` / `#if defined(__cplusplus)`;
* - the body may contain NO other preprocessor directive (a nested `#if`,
* `#else`, or `#define` bails the whole region — those need real
* preprocessing, so the file keeps its current behavior);
* - the region must close with `#endif` within a few lines (guards are
* tiny; a giant region is something else).
* The `#ifdef`/`#endif` directive lines themselves are kept — an empty
* preproc_ifdef parses clean — and every blanked byte becomes a space with
* `\r` preserved, so offsets, lines, and columns survive on CRLF checkouts.
*/
const C_CPLUSPLUS_GUARD_OPEN_RE =
/^[ \t]*#[ \t]*(?:ifdef[ \t]+__cplusplus\b|if[ \t]+defined[ \t]*\(?[ \t]*__cplusplus[ \t]*\)?)/;
const C_PREPROC_DIRECTIVE_RE = /^[ \t]*#/;
const C_PREPROC_ENDIF_RE = /^[ \t]*#[ \t]*endif\b/;
const C_CPLUSPLUS_GUARD_MAX_BODY_LINES = 40;
export function blankCCplusplusGuardBodies(source: string): string {
if (source.indexOf('__cplusplus') === -1) return source;
const lines = source.split('\n');
const stripCr = (l: string): string => (l.endsWith('\r') ? l.slice(0, -1) : l);
let changed = false;
for (let i = 0; i < lines.length; i++) {
if (!C_CPLUSPLUS_GUARD_OPEN_RE.test(stripCr(lines[i] as string))) continue;
let end = -1;
for (let j = i + 1; j < lines.length && j - i - 1 <= C_CPLUSPLUS_GUARD_MAX_BODY_LINES; j++) {
const line = stripCr(lines[j] as string);
if (C_PREPROC_ENDIF_RE.test(line)) {
end = j;
break;
}
if (C_PREPROC_DIRECTIVE_RE.test(line)) break; // nested directive — bail
}
if (end < 0) continue;
for (let k = i + 1; k < end; k++) {
lines[k] = (lines[k] as string).replace(/[^\r]/g, ' ');
}
changed = true;
i = end;
}
return changed ? lines.join('\n') : source;
}
/**
* Blank a C iterator-macro call in STATEMENT position — `ql_foreach(iter,
* &arena->tcache_ql, link) { … }` (jemalloc), `for_each_string_list_item(item,
* &list) { … }` (git), `list_for_each_entry(pos, head, member) { … }` (the
* Linux kernel's core iteration idiom). A call followed by a brace block is
* not a C statement, so tree-sitter-c drops into error recovery at every use —
* these macros are the single largest source of parse errors in macro-heavy C
* trees (git: ~39% of files error; the kernel path defers each one to wasm).
* Blanking JUST the macro call leaves the brace block as a bare compound
* statement — valid C — so the body's calls/locals extract normally on both
* arms instead of riding error recovery.
*
* C-ONLY, and matched tightly:
* - the call must be INDENTED (statement position; file-scope definitions
* start at column 0, and an unbraced file-scope `name(args) { }` is a
* valid implicit-int function definition that must not be touched);
* - lowercase-led identifier (iterator macros are lowercase by convention;
* this also excludes constructors if the file is really C++) that is not a
* control keyword;
* - the parens balance ON the line (string literals skipped), and after
* them only `{` or end-of-line may follow — a `;` (a real call statement),
* an operator, or any other token disqualifies;
* - when the line ends at `)`, the NEXT non-blank line must begin with `{`.
* C++ deliberately does NOT get this pass: an indented snake_case
* constructor (`basic_string_view(const Char* s) : … {`) is exactly this
* shape, and blanking it would corrupt every STL-style class.
*/
const C_STMT_MACRO_KEYWORDS = new Set([
'if', 'while', 'for', 'switch', 'return', 'do', 'else', 'sizeof',
]);
export function blankCStatementMacroCalls(source: string): string {
const lines = source.split('\n');
let changed = false;
const content = (l: string): string => l.replace(/\r$/, '').trim();
for (let i = 0; i < lines.length; i++) {
const line = lines[i] as string;
const m = /^[ \t]+([a-z_][a-z0-9_]*)[ \t]*\(/.exec(line);
if (!m || C_STMT_MACRO_KEYWORDS.has(m[1] as string)) continue;
const open = line.indexOf('(', m[0].length - 1);
let depth = 0;
let close = -1;
for (let k = open; k < line.length; k++) {
const ch = line[k];
if (ch === '"' || ch === "'") {
const quote = ch;
k++;
while (k < line.length && line[k] !== quote) {
if (line[k] === '\\') k++;
k++;
}
continue;
}
if (ch === '(') depth++;
else if (ch === ')') {
depth--;
if (depth === 0) {
close = k;
break;
}
}
}
if (close < 0) continue; // parens don't balance on the line
const after = line.slice(close + 1).replace(/\r$/, '').trim();
if (after === '') {
// Brace on the next line (`ql_foreach(…)\n{`) or a brace-less
// single-statement body (`for_each_subsys(ss, i)\n\tstmt;` — blanking
// leaves the bare statement, valid C). A next line starting with an
// operator/string/`;` is an expression continuation — bail.
let next = i + 1;
while (next < lines.length && content(lines[next] as string) === '') next++;
if (next >= lines.length) continue;
const first = content(lines[next] as string)[0];
if (!first || !/[A-Za-z_{]/.test(first)) continue;
} else if (after !== '{') {
continue;
}
const identStart = line.indexOf(m[1] as string);
lines[i] =
line.slice(0, identStart) +
' '.repeat(close + 1 - identStart) +
line.slice(close + 1);
changed = true;
}
return changed ? lines.join('\n') : source;
}
/**
* Blank a trailing parameter-attribute macro — `int argc UNUSED,` /
* `struct repository *repo UNUSED)` — git's house style for
* `__attribute__((unused))` on nearly every callback parameter (and the same
* shape as `MAYBE_UNUSED`/`G_GNUC_UNUSED` elsewhere). tree-sitter-c can't
* parse a second identifier after the parameter name, so every such
* SIGNATURE drops into error recovery — the single largest deferral bucket
* on git (~150 files). Blanking the macro leaves an ordinary parameter.
*
* Matched tightly: an identifier, whitespace, then an ALL-CAPS ≥3-char token
* immediately before `,` or `)`. Two juxtaposed identifiers in that position
* have no other valid-C reading — in a CALL the would-be macro is preceded
* by `,`/`(`, an operator, or a literal, never by a bare identifier. C-only:
* C++ grammars accept more juxtapositions (user-defined suffixes, macro'd
* `final`/`override`), so cpp keeps its existing recovery there.
*/
const C_TRAILING_PARAM_ATTR_RE = /\b([A-Za-z_]\w*)([ \t]+)([A-Z][A-Z0-9_]{2,})(?=[ \t]*[,)])/g;
export function blankCTrailingParamAttrMacros(source: string): string {
if (!C_TRAILING_PARAM_ATTR_RE.test(source)) {
C_TRAILING_PARAM_ATTR_RE.lastIndex = 0;
return source;
}
C_TRAILING_PARAM_ATTR_RE.lastIndex = 0;
return source.replace(
C_TRAILING_PARAM_ATTR_RE,
(_m, name: string, ws: string, macro: string) => name + ws + ' '.repeat(macro.length)
);
}
/**
* Blank the Linux-kernel/sparse declaration-annotation macros — `static int
* __init audit_init(void)`, `void __user *buf`, `__bpf_kfunc void f(…)`,
* `int x __ro_after_init;`. These lowercase double-underscore annotations sit
* between storage/type tokens and the declarator, a position tree-sitter-c
* can't reconcile, and they blanket the Linux tree: measured on the kernel's
* own `kernel/` + `mm/` subtrees, they are the largest single deferral cause
* (the `__init` family alone heads ~37% of erroring files).
*
* A structural match is IMPOSSIBLE here: `__u32 count` (a real typedef) and
* `__init foo` (an annotation) are byte-shape identical — so unlike the
* shape-keyed blanks above, this is a CURATED list (the CPP_INLINE_MACROS
* precedent) of well-known sparse/section/compiler annotations that are
* reserved-namespace macros in every codebase that spells them. Whole-word,
* equal-length spaces, C-only (the C++ grammar's kernel exposure is
* negligible and cpp keeps its narrower blank set).
*/
const C_KERNEL_ANNOTATIONS = [
'__init', '__exit', '__initdata', '__initconst', '__exitdata',
'__devinit', '__devexit', '__cpuinit', '__meminit', '__meminitdata',
'__net_init', '__net_exit', '__init_or_module',
'__user', '__kernel', '__iomem', '__percpu', '__rcu', '__force', '__nocast',
'__must_check', '__maybe_unused', '__always_unused', '__used', '__cold',
'__hot', '__weak', '__pure', '__sched', '__malloc', '__visible',
'__deprecated', '__ro_after_init', '__read_mostly', '__refdata',
'__latent_entropy', '__randomize_layout', '__no_randomize_layout',
'__bpf_kfunc', '__function_aligned', '__always_inline', '__noreturn',
] as const;
// `(?!\s*\()` keeps the parameterized annotations (`__printf(1, 2)`,
// `__aligned(8)`, `__section("x")`) intact — blanking just their name would
// strand the argument list as a floating parenthesis and CREATE an error.
const C_KERNEL_ANNOTATION_RE = new RegExp(
`\\b(${[...C_KERNEL_ANNOTATIONS].sort((a, b) => b.length - a.length).join('|')})\\b(?!\\s*\\()`,
'g'
);
export function blankCKernelAnnotations(source: string): string {
if (source.indexOf('__') === -1) return source;
C_KERNEL_ANNOTATION_RE.lastIndex = 0;
if (!C_KERNEL_ANNOTATION_RE.test(source)) return source;
let out = source.replace(C_KERNEL_ANNOTATION_RE, (m) => ' '.repeat(m.length));
// `container_of(ptr, struct T, member)` — the type-keyword argument is the
// one call shape tree-sitter-c cannot read (a macro taking a TYPE), and it
// is pervasive across the Linux tree. Blanking just the `struct`/`union`
// keyword leaves `container_of(ptr, T, member)` — a plain
// identifier argument the grammar parses natively. Keyed to the macro name
// so no other `struct` keyword anywhere is ever touched.
if (out.indexOf('container_of') !== -1) {
out = out.replace(
/(\bcontainer_of\s*\([^;()]*?,\s*)(struct|union)(\s+)/g,
(_m, head: string, kw: string, ws: string) => head + ' '.repeat(kw.length) + ws
);
}
return out;
}
/** C source pre-processing: neutralize `#ifdef __cplusplus` compat-guard
* bodies (invisible to a C compiler; `extern "C" {` otherwise errors every
* public header), blank declaration-markup macro calls and lone macro lines
* (`REDIS_NO_SANITIZE("bounds")` before a definition, jemalloc's diagnostic
* toggles — the same structural shapes the C++ side already blanks), recover
* functions hidden behind a leading attribute macro (#1211), then — for
* C-detected headers in CUDA projects (llm.c keeps `__device__` helpers and
* kernel prototypes in plain `.h`) — the same content-gated CUDA blank as
* C++. Offset-preserving. */
function preParseCSource(source: string): string {
const blanked = blankCLeadingAttrMacros(source);
return looksLikeCudaSource(blanked) ? blankCudaConstructs(blanked) : blanked;
let blanked = blankCLeadingAttrMacros(
blankLoneMacroLines(
blankCStatementMacroCalls(
blankCTrailingParamAttrMacros(
blankCppAnnotationMacroCalls(
blankCKernelAnnotations(blankCCplusplusGuardBodies(source))
)
)
)
)
);
if (looksLikeCudaSource(blanked)) blanked = blankCudaConstructs(blanked);
return restoreDirectiveLines(source, blanked);
}
export const cppExtractor: LanguageExtractor = {
+24 -5
View File
@@ -30,7 +30,7 @@ import { DfmExtractor } from './dfm-extractor';
import { VueExtractor } from './vue-extractor';
import { MyBatisExtractor } from './mybatis-extractor';
import { CfmlExtractor } from './cfml-extractor';
import { tryKernelExtract } from './kernel';
import { tryKernelExtract, takeDeferredPreParse } from './kernel';
import {
getAllFrameworkResolvers,
getApplicableFrameworks,
@@ -429,13 +429,23 @@ export class TreeSitterExtractor {
private fnRefCandidates: Array<FnRefCandidate & { fromNodeId: string }> = [];
// Memoized "is this a Vue store file" verdict (per-extractor = per-file).
private vueStoreFile: boolean | null = null;
// Source already went through the extractor's preParse at the kernel route
// point (this instance is the wasm fallback for a kernel-deferred file) —
// don't blank it a second time.
private sourceIsPreParsed = false;
constructor(filePath: string, source: string, language?: Language) {
constructor(
filePath: string,
source: string,
language?: Language,
options?: { sourceIsPreParsed?: boolean }
) {
this.filePath = filePath;
this.source = source;
this.language = language || detectLanguage(filePath, source);
this.extractor = EXTRACTORS[this.language] || null;
this.fnRefSpec = FN_REF_SPECS[this.language];
this.sourceIsPreParsed = options?.sourceIsPreParsed === true;
}
/**
@@ -484,8 +494,9 @@ export class TreeSitterExtractor {
// grammar gaps — e.g. C# blanks conditional-compilation directive lines
// the grammar mis-parses inside enum bodies (#237). We reassign
// this.source so downstream getNodeText reads the same bytes the parser
// saw (identical outside the blanked directive lines).
if (this.extractor?.preParse) {
// saw (identical outside the blanked directive lines). Skipped when the
// kernel route point already applied it (sourceIsPreParsed).
if (this.extractor?.preParse && !this.sourceIsPreParsed) {
this.source = this.extractor.preParse(this.source, this.filePath);
}
this.tree = parser.parse(this.source) ?? null;
@@ -6708,7 +6719,15 @@ export function extractFromSource(
if (kernelResult) {
result = kernelResult;
} else {
const extractor = new TreeSitterExtractor(filePath, source, detectedLanguage);
// A kernel-deferred file already paid the (offset-preserving) preParse
// at the route point — reuse those bytes instead of blanking again.
const deferredPre = takeDeferredPreParse(filePath, source, detectedLanguage);
const extractor = new TreeSitterExtractor(
filePath,
deferredPre ?? source,
detectedLanguage,
{ sourceIsPreParsed: deferredPre != null }
);
result = extractor.extract();
}
}