feat(kernel): R7a C/C++ walker — dual-lang ccpp module, preParse hoist, 7 new blanks, c/cpp default-routed (#1346)
Parity: 0 diffs on redis/git/fmt/protobuf/ALS sweeps; full-init dumps byte-identical on all five + linux at kernel scale (10.4M dump lines, same sha256 both arms). Linux 2c/6GB envelope: kernel-arm 19.1min vs wasm-arm 22.9min (parse 356s vs 435s) on a much richer graph (the new blanks recover error-swallowed code: git 2x nodes, linux kernel/+mm/ 3x). Deferral guard corrected by measurement (C/C++ error incidence 9-42%; --max-deferral flag); defer-reuse memo kills the 3x re-blank/re-parse cost deferred files paid. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
44561b6aad
commit
2d72891b59
@@ -15,6 +15,7 @@
|
||||
*/
|
||||
|
||||
import type { ExtractionResult, Language } from '../../types';
|
||||
import { EXTRACTORS } from '../languages';
|
||||
import { getKernel, kernelSupports } from './loader';
|
||||
import { decodeExtractBuffers } from './decode';
|
||||
import {
|
||||
@@ -41,6 +42,12 @@ const DEFAULT_ROUTED: ReadonlySet<Language> = new Set<Language>([
|
||||
'java',
|
||||
'python',
|
||||
'go',
|
||||
// R7a (2026-07-17): parity swept 0-diff on redis/git/fmt/protobuf/ALS
|
||||
// (2,389 files compared) + full-init dump-diffs byte-identical; erroring
|
||||
// files defer per-file to wasm (routine for macro-heavy C/C++ — see
|
||||
// scripts/kernel-parity.mjs --max-deferral).
|
||||
'c',
|
||||
'cpp',
|
||||
]);
|
||||
|
||||
/**
|
||||
@@ -55,6 +62,22 @@ const POST_PASSES: Partial<Record<Language, KernelPostPass>> = {
|
||||
// (none yet — R2+)
|
||||
};
|
||||
|
||||
/**
|
||||
* The preParse hoist (checklist §arch-1): languages with an offset-preserving
|
||||
* `preParse` hook (c/cpp macro blanking, csharp #237, metal #1121, cuda #1172)
|
||||
* apply it HERE, before the kernel call, so both arms parse identical blanked
|
||||
* bytes and none of the blanking logic needs a Rust port. The wasm fallback
|
||||
* path is untouched — TreeSitterExtractor applies the same hook itself on the
|
||||
* RAW source it receives, so a kernel error/defer still extracts identically.
|
||||
* Every blank is an equal-length-space replacement, so offsets, lines, and
|
||||
* columns survive; `filePath` rides along for the extension-gated dialect
|
||||
* blanks (`.metal` attributes; `.cu`/`.cuh` + content-gated CUDA).
|
||||
*/
|
||||
function preParsedSource(filePath: string, source: string, language: Language): string {
|
||||
const pre = EXTRACTORS[language]?.preParse;
|
||||
return pre ? pre(source, filePath) : source;
|
||||
}
|
||||
|
||||
function isRouted(language: Language): boolean {
|
||||
const env = process.env.CODEGRAPH_KERNEL_LANGS;
|
||||
if (env === undefined || env === '') return DEFAULT_ROUTED.has(language);
|
||||
@@ -73,6 +96,37 @@ export function kernelRoutes(language: Language): boolean {
|
||||
/** Warned-once registry so a broken language logs a single line, not one per file. */
|
||||
const warned = new Set<string>();
|
||||
|
||||
/**
|
||||
* One-slot defer memo. A file the kernel defers (parse errors → wasm) used to
|
||||
* pay the full pipeline again at every seam: the worker's raw try blanked +
|
||||
* native-parsed it, extractFromSource's kernel try blanked + native-parsed it
|
||||
* AGAIN, and the wasm extractor then re-applied preParse a third time. On a
|
||||
* high-deferral tree (the Linux kernel defers ~79% of files) that waste
|
||||
* dominated the arm's parse phase. The slot remembers the LAST deferred
|
||||
* (file, source, language) so (a) a repeat kernel attempt for the same file
|
||||
* short-circuits to null, and (b) the wasm fallback can reuse the
|
||||
* already-blanked source instead of re-running preParse. Source is matched by
|
||||
* string identity — the worker passes the same string through every seam.
|
||||
*/
|
||||
let deferSlot: { filePath: string; source: string; language: Language; pre: string } | null = null;
|
||||
|
||||
/** The hoisted preParse output for a just-deferred file, if it matches. */
|
||||
export function takeDeferredPreParse(
|
||||
filePath: string,
|
||||
source: string,
|
||||
language: Language
|
||||
): string | null {
|
||||
if (
|
||||
deferSlot &&
|
||||
deferSlot.filePath === filePath &&
|
||||
deferSlot.source === source &&
|
||||
deferSlot.language === language
|
||||
) {
|
||||
return deferSlot.pre;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The raw table buffers + the cheap facts the orchestrator needs pre-decode. */
|
||||
export interface KernelRawResult {
|
||||
buffers: NonNullable<ExtractionResult['kernelBuffers']>;
|
||||
@@ -96,8 +150,10 @@ export function tryKernelExtractRaw(
|
||||
if (!kernelRoutes(language) || POST_PASSES[language]) return null;
|
||||
const kernel = getKernel();
|
||||
if (!kernel) return null;
|
||||
if (takeDeferredPreParse(filePath, source, language) !== null) return null; // already deferred
|
||||
const pre = preParsedSource(filePath, source, language);
|
||||
try {
|
||||
const buffers = kernel.extractFile(filePath, source, language);
|
||||
const buffers = kernel.extractFile(filePath, pre, language);
|
||||
const meta = buffers.meta;
|
||||
if (meta.readUInt8(LAYOUT_META.version) !== LAYOUT_ABI) {
|
||||
throw new Error(`kernel buffer ABI ${meta.readUInt8(0)} != expected ${LAYOUT_ABI}`);
|
||||
@@ -118,7 +174,10 @@ export function tryKernelExtractRaw(
|
||||
return { buffers, counts, errors };
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
if (message.includes('defer:')) return null;
|
||||
if (message.includes('defer:')) {
|
||||
deferSlot = { filePath, source, language, pre };
|
||||
return null;
|
||||
}
|
||||
if (!warned.has(language)) {
|
||||
warned.add(language);
|
||||
process.stderr.write(
|
||||
@@ -166,13 +225,11 @@ export function tryKernelExtract(
|
||||
if (!kernelRoutes(language)) return null;
|
||||
const kernel = getKernel();
|
||||
if (!kernel) return null;
|
||||
if (takeDeferredPreParse(filePath, source, language) !== null) return null; // already deferred
|
||||
const t0 = Date.now();
|
||||
const pre = preParsedSource(filePath, source, language);
|
||||
try {
|
||||
// NOTE(T2 languages): when a preParse-carrying language (csharp #237,
|
||||
// metal #1121, cuda #1172, c/cpp macro blanking) routes here, its
|
||||
// offset-preserving preParse hook must be applied to `source` first —
|
||||
// wire that alongside the language's port, gated WITH its equivalence run.
|
||||
const buffers = kernel.extractFile(filePath, source, language);
|
||||
const buffers = kernel.extractFile(filePath, pre, language);
|
||||
const result = decodeExtractBuffers(buffers, filePath, language);
|
||||
POST_PASSES[language]?.(result, source);
|
||||
result.durationMs = Date.now() - t0;
|
||||
@@ -182,7 +239,10 @@ export function tryKernelExtract(
|
||||
// `defer:` is the kernel's expected-routing signal (files with parse
|
||||
// errors take the wasm path — its error RECOVERY is the canonical one;
|
||||
// recovery differs between UTF-8 and UTF-16 parsing). Silent by design.
|
||||
if (message.includes('defer:')) return null;
|
||||
if (message.includes('defer:')) {
|
||||
deferSlot = { filePath, source, language, pre };
|
||||
return null;
|
||||
}
|
||||
if (!warned.has(language)) {
|
||||
warned.add(language);
|
||||
process.stderr.write(
|
||||
|
||||
@@ -520,6 +520,63 @@ export function blankCppAnnotationMacroCalls(source: string): string {
|
||||
return chars.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Blank a macro that is the ONLY token on its line — no parens, no semicolon:
|
||||
* namespace-management macros (`FMT_BEGIN_NAMESPACE`, `FMT_END_EXPORT`,
|
||||
* `JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS`), Qt's `Q_OBJECT`, and friends. A
|
||||
* bare identifier is not a statement or declaration in C or C++, so
|
||||
* tree-sitter drops into error recovery at every one — and since the kernel
|
||||
* path defers ANY erroring file to wasm, this single idiom deferred 13/73 fmt
|
||||
* files and a comparable share of jemalloc, forfeiting the native-parse win
|
||||
* on exactly the header-heavy trees it targets (the wasm path also mis-nests
|
||||
* scopes around them today). Replacing the token with equal-length spaces
|
||||
* preserves every byte offset and the surrounding declarations parse clean.
|
||||
*
|
||||
* Matched tightly so a real identifier can never be touched — ALL of:
|
||||
* - the line consists of ONE ALL-CAPS token (≥4 chars, with `_`), optionally
|
||||
* followed by a same-line comment — a lone lowercase identifier or any
|
||||
* second token disqualifies;
|
||||
* - the PREVIOUS non-blank line does not end in a continuation character
|
||||
* (`=`, an operator, `,`, `(`, `?`, `:`, or a `\` macro-definition
|
||||
* continuation) — so an ALL-CAPS operand split onto its own line inside a
|
||||
* multi-line expression (`int x =\n SOME_CONST\n | OTHER;`) is left
|
||||
* alone; and
|
||||
* - the NEXT non-blank line starts like a declaration/scope token
|
||||
* (letter, `_`, `#`, `{`, `}`, or `~`) or the file ends — an operator,
|
||||
* string literal, or `;` continuation rejects the match.
|
||||
* Shared by C and C++ (the idiom is identical in both).
|
||||
*/
|
||||
const LONE_MACRO_LINE_RE = /^[ \t]*([A-Z][A-Z0-9_]{3,})[ \t]*(?:\/\/[^\n\r]*|\/\*[^\n\r]*\*\/[ \t]*)?\r?$/;
|
||||
const LONE_MACRO_CONTINUATION_END_RE = /[=+\-*/%&|^<>?:,(\\]$/;
|
||||
export function blankLoneMacroLines(source: string): string {
|
||||
if (!/^[ \t]*[A-Z][A-Z0-9_]{3,}[ \t]*\r?$/m.test(source)) return source;
|
||||
const lines = source.split('\n');
|
||||
const content = (l: string): string => l.replace(/\r$/, '').trim();
|
||||
let changed = false;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i] as string;
|
||||
const m = LONE_MACRO_LINE_RE.exec(line);
|
||||
if (!m) continue;
|
||||
// Underscore requirement rides the macro convention (FMT_BEGIN_NAMESPACE,
|
||||
// Q_OBJECT); a solid all-caps word (`NDEBUG`-style) alone is too risky.
|
||||
if (!(m[1] as string).includes('_')) continue;
|
||||
let prev = i - 1;
|
||||
while (prev >= 0 && content(lines[prev] as string) === '') prev--;
|
||||
if (prev >= 0 && LONE_MACRO_CONTINUATION_END_RE.test(content(lines[prev] as string))) continue;
|
||||
let next = i + 1;
|
||||
while (next < lines.length && content(lines[next] as string) === '') next++;
|
||||
if (next < lines.length) {
|
||||
const first = content(lines[next] as string)[0];
|
||||
if (!first || !/[A-Za-z_#{}~]/.test(first)) continue;
|
||||
}
|
||||
const start = line.indexOf(m[1] as string);
|
||||
lines[i] =
|
||||
line.slice(0, start) + ' '.repeat((m[1] as string).length) + line.slice(start + (m[1] as string).length);
|
||||
changed = true;
|
||||
}
|
||||
return changed ? lines.join('\n') : source;
|
||||
}
|
||||
|
||||
/**
|
||||
* Blank an export/visibility macro sitting in front of a *member* or *method*
|
||||
* declaration inside a class/namespace (`ENGINE_API virtual void Tick(…)`,
|
||||
@@ -689,24 +746,64 @@ function looksLikeCudaSource(source: string): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore preprocessor-directive lines to their original bytes after the
|
||||
* blanking passes ran. The token-level blanks match on shape, not context, so
|
||||
* a macro name that happens to sit inside a DIRECTIVE gets blanked too — and
|
||||
* blanking the name position of `#define FMT_API FMT_VISIBILITY("default")`
|
||||
* leaves a nameless `# define FMT_VISIBILITY(…)`, which is a parse
|
||||
* ERROR (fmt's base.h carries several). Inside a directive the blanks were
|
||||
* never useful anyway: tree-sitter stores `#define` bodies as raw
|
||||
* preproc_arg text it doesn't parse, so blanking there can only ever break
|
||||
* the directive itself. Copying the original directive lines back (including
|
||||
* `\`-continuation lines of multi-line defines) is offset-preserving by
|
||||
* construction and strictly reduces parse errors on both extraction arms.
|
||||
*/
|
||||
function restoreDirectiveLines(original: string, blanked: string): string {
|
||||
if (blanked === original || original.indexOf('#') === -1) return blanked;
|
||||
const o = original.split('\n');
|
||||
const b = blanked.split('\n');
|
||||
let changed = false;
|
||||
let continuation: boolean = false;
|
||||
for (let i = 0; i < o.length && i < b.length; i++) {
|
||||
const line = o[i] as string;
|
||||
const isDirective: boolean = continuation || /^[ \t]*#/.test(line);
|
||||
if (isDirective && b[i] !== line) {
|
||||
b[i] = line;
|
||||
changed = true;
|
||||
}
|
||||
continuation = isDirective && /\\\s*$/.test(line.replace(/\r$/, ''));
|
||||
}
|
||||
return changed ? b.join('\n') : blanked;
|
||||
}
|
||||
|
||||
/** C/C++ source pre-processing before tree-sitter: recover macro-annotated class
|
||||
* definitions, macro-prefixed function definitions, macro-prefixed members, and
|
||||
* macro-decorated members (Unreal-Engine reflection markup) — plus the non-C++
|
||||
* surface of the dialects parsed with the C++ grammar: `.metal` MSL attribute
|
||||
* annotations, and CUDA specifiers + launch syntax (by `.cu`/`.cuh` extension
|
||||
* or by content, for CUDA living in `.h`/`.hpp` headers). Offset-preserving. */
|
||||
* or by content, for CUDA living in `.h`/`.hpp` headers). Offset-preserving;
|
||||
* directive lines are restored at the end (see restoreDirectiveLines). */
|
||||
function preParseCppSource(source: string, filePath?: string): string {
|
||||
const blanked = blankCppAnnotationMacroCalls(
|
||||
blankCppInlineAnnotationMacros(
|
||||
blankCppApiPrefixMacros(blankCppInlineMacros(blankCppExportMacros(source)))
|
||||
// blankCLeadingAttrMacros runs AFTER the api-prefix blank so a stacked
|
||||
// `FMT_NORETURN FMT_API void f(…)` reduces to the `MACRO Ret name(` shape
|
||||
// it matches (the _API token is already spaces by then).
|
||||
let blanked = blankLoneMacroLines(
|
||||
blankCLeadingAttrMacros(
|
||||
blankCppAnnotationMacroCalls(
|
||||
blankCppInlineAnnotationMacros(
|
||||
blankCppApiPrefixMacros(blankCppInlineMacros(blankCppExportMacros(source)))
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
const lower = filePath ? filePath.toLowerCase() : '';
|
||||
if (lower.endsWith('.metal')) return blankMetalAttributes(blanked);
|
||||
if (lower.endsWith('.cu') || lower.endsWith('.cuh') || looksLikeCudaSource(source)) {
|
||||
return blankCudaConstructs(blanked);
|
||||
if (lower.endsWith('.metal')) {
|
||||
blanked = blankMetalAttributes(blanked);
|
||||
} else if (lower.endsWith('.cu') || lower.endsWith('.cuh') || looksLikeCudaSource(source)) {
|
||||
blanked = blankCudaConstructs(blanked);
|
||||
}
|
||||
return blanked;
|
||||
return restoreDirectiveLines(source, blanked);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -743,13 +840,259 @@ export function blankCLeadingAttrMacros(source: string): string {
|
||||
);
|
||||
}
|
||||
|
||||
/** C source pre-processing: recover functions hidden behind a leading
|
||||
* attribute macro (#1211), then — for C-detected headers in CUDA projects
|
||||
* (llm.c keeps `__device__` helpers and kernel prototypes in plain `.h`) —
|
||||
* the same content-gated CUDA blank as C++. Offset-preserving. */
|
||||
/**
|
||||
* Blank the body of `#ifdef __cplusplus … #endif` guard regions in C sources.
|
||||
* The ubiquitous C-header compatibility idiom
|
||||
*
|
||||
* #ifdef __cplusplus
|
||||
* extern "C" {
|
||||
* #endif
|
||||
*
|
||||
* is NOT valid C — `extern "C" {` (and any other C++-only line under the
|
||||
* guard) drops tree-sitter-c into error recovery, so effectively every public
|
||||
* C header carries parse errors. The wasm path shrugs (recovery keeps the
|
||||
* rest); the kernel path defers EVERY erroring file to wasm by policy — so
|
||||
* this one idiom pushed C-header deferral to ~32% on redis (vs the <10%
|
||||
* gate) and forfeited the native-parse win exactly where C repos have the
|
||||
* most files. A C compiler never sees the guarded lines (`__cplusplus` is
|
||||
* only defined for C++), so blanking the region BODY mirrors the
|
||||
* preprocessor's own view of the file.
|
||||
*
|
||||
* Matched conservatively, line-based and offset-preserving:
|
||||
* - the opener must be `#ifdef __cplusplus` / `#if defined(__cplusplus)`;
|
||||
* - the body may contain NO other preprocessor directive (a nested `#if`,
|
||||
* `#else`, or `#define` bails the whole region — those need real
|
||||
* preprocessing, so the file keeps its current behavior);
|
||||
* - the region must close with `#endif` within a few lines (guards are
|
||||
* tiny; a giant region is something else).
|
||||
* The `#ifdef`/`#endif` directive lines themselves are kept — an empty
|
||||
* preproc_ifdef parses clean — and every blanked byte becomes a space with
|
||||
* `\r` preserved, so offsets, lines, and columns survive on CRLF checkouts.
|
||||
*/
|
||||
const C_CPLUSPLUS_GUARD_OPEN_RE =
|
||||
/^[ \t]*#[ \t]*(?:ifdef[ \t]+__cplusplus\b|if[ \t]+defined[ \t]*\(?[ \t]*__cplusplus[ \t]*\)?)/;
|
||||
const C_PREPROC_DIRECTIVE_RE = /^[ \t]*#/;
|
||||
const C_PREPROC_ENDIF_RE = /^[ \t]*#[ \t]*endif\b/;
|
||||
const C_CPLUSPLUS_GUARD_MAX_BODY_LINES = 40;
|
||||
export function blankCCplusplusGuardBodies(source: string): string {
|
||||
if (source.indexOf('__cplusplus') === -1) return source;
|
||||
const lines = source.split('\n');
|
||||
const stripCr = (l: string): string => (l.endsWith('\r') ? l.slice(0, -1) : l);
|
||||
let changed = false;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (!C_CPLUSPLUS_GUARD_OPEN_RE.test(stripCr(lines[i] as string))) continue;
|
||||
let end = -1;
|
||||
for (let j = i + 1; j < lines.length && j - i - 1 <= C_CPLUSPLUS_GUARD_MAX_BODY_LINES; j++) {
|
||||
const line = stripCr(lines[j] as string);
|
||||
if (C_PREPROC_ENDIF_RE.test(line)) {
|
||||
end = j;
|
||||
break;
|
||||
}
|
||||
if (C_PREPROC_DIRECTIVE_RE.test(line)) break; // nested directive — bail
|
||||
}
|
||||
if (end < 0) continue;
|
||||
for (let k = i + 1; k < end; k++) {
|
||||
lines[k] = (lines[k] as string).replace(/[^\r]/g, ' ');
|
||||
}
|
||||
changed = true;
|
||||
i = end;
|
||||
}
|
||||
return changed ? lines.join('\n') : source;
|
||||
}
|
||||
|
||||
/**
|
||||
* Blank a C iterator-macro call in STATEMENT position — `ql_foreach(iter,
|
||||
* &arena->tcache_ql, link) { … }` (jemalloc), `for_each_string_list_item(item,
|
||||
* &list) { … }` (git), `list_for_each_entry(pos, head, member) { … }` (the
|
||||
* Linux kernel's core iteration idiom). A call followed by a brace block is
|
||||
* not a C statement, so tree-sitter-c drops into error recovery at every use —
|
||||
* these macros are the single largest source of parse errors in macro-heavy C
|
||||
* trees (git: ~39% of files error; the kernel path defers each one to wasm).
|
||||
* Blanking JUST the macro call leaves the brace block as a bare compound
|
||||
* statement — valid C — so the body's calls/locals extract normally on both
|
||||
* arms instead of riding error recovery.
|
||||
*
|
||||
* C-ONLY, and matched tightly:
|
||||
* - the call must be INDENTED (statement position; file-scope definitions
|
||||
* start at column 0, and an unbraced file-scope `name(args) { }` is a
|
||||
* valid implicit-int function definition that must not be touched);
|
||||
* - lowercase-led identifier (iterator macros are lowercase by convention;
|
||||
* this also excludes constructors if the file is really C++) that is not a
|
||||
* control keyword;
|
||||
* - the parens balance ON the line (string literals skipped), and after
|
||||
* them only `{` or end-of-line may follow — a `;` (a real call statement),
|
||||
* an operator, or any other token disqualifies;
|
||||
* - when the line ends at `)`, the NEXT non-blank line must begin with `{`.
|
||||
* C++ deliberately does NOT get this pass: an indented snake_case
|
||||
* constructor (`basic_string_view(const Char* s) : … {`) is exactly this
|
||||
* shape, and blanking it would corrupt every STL-style class.
|
||||
*/
|
||||
const C_STMT_MACRO_KEYWORDS = new Set([
|
||||
'if', 'while', 'for', 'switch', 'return', 'do', 'else', 'sizeof',
|
||||
]);
|
||||
export function blankCStatementMacroCalls(source: string): string {
|
||||
const lines = source.split('\n');
|
||||
let changed = false;
|
||||
const content = (l: string): string => l.replace(/\r$/, '').trim();
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i] as string;
|
||||
const m = /^[ \t]+([a-z_][a-z0-9_]*)[ \t]*\(/.exec(line);
|
||||
if (!m || C_STMT_MACRO_KEYWORDS.has(m[1] as string)) continue;
|
||||
const open = line.indexOf('(', m[0].length - 1);
|
||||
let depth = 0;
|
||||
let close = -1;
|
||||
for (let k = open; k < line.length; k++) {
|
||||
const ch = line[k];
|
||||
if (ch === '"' || ch === "'") {
|
||||
const quote = ch;
|
||||
k++;
|
||||
while (k < line.length && line[k] !== quote) {
|
||||
if (line[k] === '\\') k++;
|
||||
k++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (ch === '(') depth++;
|
||||
else if (ch === ')') {
|
||||
depth--;
|
||||
if (depth === 0) {
|
||||
close = k;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (close < 0) continue; // parens don't balance on the line
|
||||
const after = line.slice(close + 1).replace(/\r$/, '').trim();
|
||||
if (after === '') {
|
||||
// Brace on the next line (`ql_foreach(…)\n{`) or a brace-less
|
||||
// single-statement body (`for_each_subsys(ss, i)\n\tstmt;` — blanking
|
||||
// leaves the bare statement, valid C). A next line starting with an
|
||||
// operator/string/`;` is an expression continuation — bail.
|
||||
let next = i + 1;
|
||||
while (next < lines.length && content(lines[next] as string) === '') next++;
|
||||
if (next >= lines.length) continue;
|
||||
const first = content(lines[next] as string)[0];
|
||||
if (!first || !/[A-Za-z_{]/.test(first)) continue;
|
||||
} else if (after !== '{') {
|
||||
continue;
|
||||
}
|
||||
const identStart = line.indexOf(m[1] as string);
|
||||
lines[i] =
|
||||
line.slice(0, identStart) +
|
||||
' '.repeat(close + 1 - identStart) +
|
||||
line.slice(close + 1);
|
||||
changed = true;
|
||||
}
|
||||
return changed ? lines.join('\n') : source;
|
||||
}
|
||||
|
||||
/**
|
||||
* Blank a trailing parameter-attribute macro — `int argc UNUSED,` /
|
||||
* `struct repository *repo UNUSED)` — git's house style for
|
||||
* `__attribute__((unused))` on nearly every callback parameter (and the same
|
||||
* shape as `MAYBE_UNUSED`/`G_GNUC_UNUSED` elsewhere). tree-sitter-c can't
|
||||
* parse a second identifier after the parameter name, so every such
|
||||
* SIGNATURE drops into error recovery — the single largest deferral bucket
|
||||
* on git (~150 files). Blanking the macro leaves an ordinary parameter.
|
||||
*
|
||||
* Matched tightly: an identifier, whitespace, then an ALL-CAPS ≥3-char token
|
||||
* immediately before `,` or `)`. Two juxtaposed identifiers in that position
|
||||
* have no other valid-C reading — in a CALL the would-be macro is preceded
|
||||
* by `,`/`(`, an operator, or a literal, never by a bare identifier. C-only:
|
||||
* C++ grammars accept more juxtapositions (user-defined suffixes, macro'd
|
||||
* `final`/`override`), so cpp keeps its existing recovery there.
|
||||
*/
|
||||
const C_TRAILING_PARAM_ATTR_RE = /\b([A-Za-z_]\w*)([ \t]+)([A-Z][A-Z0-9_]{2,})(?=[ \t]*[,)])/g;
|
||||
export function blankCTrailingParamAttrMacros(source: string): string {
|
||||
if (!C_TRAILING_PARAM_ATTR_RE.test(source)) {
|
||||
C_TRAILING_PARAM_ATTR_RE.lastIndex = 0;
|
||||
return source;
|
||||
}
|
||||
C_TRAILING_PARAM_ATTR_RE.lastIndex = 0;
|
||||
return source.replace(
|
||||
C_TRAILING_PARAM_ATTR_RE,
|
||||
(_m, name: string, ws: string, macro: string) => name + ws + ' '.repeat(macro.length)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Blank the Linux-kernel/sparse declaration-annotation macros — `static int
|
||||
* __init audit_init(void)`, `void __user *buf`, `__bpf_kfunc void f(…)`,
|
||||
* `int x __ro_after_init;`. These lowercase double-underscore annotations sit
|
||||
* between storage/type tokens and the declarator, a position tree-sitter-c
|
||||
* can't reconcile, and they blanket the Linux tree: measured on the kernel's
|
||||
* own `kernel/` + `mm/` subtrees, they are the largest single deferral cause
|
||||
* (the `__init` family alone heads ~37% of erroring files).
|
||||
*
|
||||
* A structural match is IMPOSSIBLE here: `__u32 count` (a real typedef) and
|
||||
* `__init foo` (an annotation) are byte-shape identical — so unlike the
|
||||
* shape-keyed blanks above, this is a CURATED list (the CPP_INLINE_MACROS
|
||||
* precedent) of well-known sparse/section/compiler annotations that are
|
||||
* reserved-namespace macros in every codebase that spells them. Whole-word,
|
||||
* equal-length spaces, C-only (the C++ grammar's kernel exposure is
|
||||
* negligible and cpp keeps its narrower blank set).
|
||||
*/
|
||||
const C_KERNEL_ANNOTATIONS = [
|
||||
'__init', '__exit', '__initdata', '__initconst', '__exitdata',
|
||||
'__devinit', '__devexit', '__cpuinit', '__meminit', '__meminitdata',
|
||||
'__net_init', '__net_exit', '__init_or_module',
|
||||
'__user', '__kernel', '__iomem', '__percpu', '__rcu', '__force', '__nocast',
|
||||
'__must_check', '__maybe_unused', '__always_unused', '__used', '__cold',
|
||||
'__hot', '__weak', '__pure', '__sched', '__malloc', '__visible',
|
||||
'__deprecated', '__ro_after_init', '__read_mostly', '__refdata',
|
||||
'__latent_entropy', '__randomize_layout', '__no_randomize_layout',
|
||||
'__bpf_kfunc', '__function_aligned', '__always_inline', '__noreturn',
|
||||
] as const;
|
||||
// `(?!\s*\()` keeps the parameterized annotations (`__printf(1, 2)`,
|
||||
// `__aligned(8)`, `__section("x")`) intact — blanking just their name would
|
||||
// strand the argument list as a floating parenthesis and CREATE an error.
|
||||
const C_KERNEL_ANNOTATION_RE = new RegExp(
|
||||
`\\b(${[...C_KERNEL_ANNOTATIONS].sort((a, b) => b.length - a.length).join('|')})\\b(?!\\s*\\()`,
|
||||
'g'
|
||||
);
|
||||
export function blankCKernelAnnotations(source: string): string {
|
||||
if (source.indexOf('__') === -1) return source;
|
||||
C_KERNEL_ANNOTATION_RE.lastIndex = 0;
|
||||
if (!C_KERNEL_ANNOTATION_RE.test(source)) return source;
|
||||
let out = source.replace(C_KERNEL_ANNOTATION_RE, (m) => ' '.repeat(m.length));
|
||||
// `container_of(ptr, struct T, member)` — the type-keyword argument is the
|
||||
// one call shape tree-sitter-c cannot read (a macro taking a TYPE), and it
|
||||
// is pervasive across the Linux tree. Blanking just the `struct`/`union`
|
||||
// keyword leaves `container_of(ptr, T, member)` — a plain
|
||||
// identifier argument the grammar parses natively. Keyed to the macro name
|
||||
// so no other `struct` keyword anywhere is ever touched.
|
||||
if (out.indexOf('container_of') !== -1) {
|
||||
out = out.replace(
|
||||
/(\bcontainer_of\s*\([^;()]*?,\s*)(struct|union)(\s+)/g,
|
||||
(_m, head: string, kw: string, ws: string) => head + ' '.repeat(kw.length) + ws
|
||||
);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** C source pre-processing: neutralize `#ifdef __cplusplus` compat-guard
|
||||
* bodies (invisible to a C compiler; `extern "C" {` otherwise errors every
|
||||
* public header), blank declaration-markup macro calls and lone macro lines
|
||||
* (`REDIS_NO_SANITIZE("bounds")` before a definition, jemalloc's diagnostic
|
||||
* toggles — the same structural shapes the C++ side already blanks), recover
|
||||
* functions hidden behind a leading attribute macro (#1211), then — for
|
||||
* C-detected headers in CUDA projects (llm.c keeps `__device__` helpers and
|
||||
* kernel prototypes in plain `.h`) — the same content-gated CUDA blank as
|
||||
* C++. Offset-preserving. */
|
||||
function preParseCSource(source: string): string {
|
||||
const blanked = blankCLeadingAttrMacros(source);
|
||||
return looksLikeCudaSource(blanked) ? blankCudaConstructs(blanked) : blanked;
|
||||
let blanked = blankCLeadingAttrMacros(
|
||||
blankLoneMacroLines(
|
||||
blankCStatementMacroCalls(
|
||||
blankCTrailingParamAttrMacros(
|
||||
blankCppAnnotationMacroCalls(
|
||||
blankCKernelAnnotations(blankCCplusplusGuardBodies(source))
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
if (looksLikeCudaSource(blanked)) blanked = blankCudaConstructs(blanked);
|
||||
return restoreDirectiveLines(source, blanked);
|
||||
}
|
||||
|
||||
export const cppExtractor: LanguageExtractor = {
|
||||
|
||||
@@ -30,7 +30,7 @@ import { DfmExtractor } from './dfm-extractor';
|
||||
import { VueExtractor } from './vue-extractor';
|
||||
import { MyBatisExtractor } from './mybatis-extractor';
|
||||
import { CfmlExtractor } from './cfml-extractor';
|
||||
import { tryKernelExtract } from './kernel';
|
||||
import { tryKernelExtract, takeDeferredPreParse } from './kernel';
|
||||
import {
|
||||
getAllFrameworkResolvers,
|
||||
getApplicableFrameworks,
|
||||
@@ -429,13 +429,23 @@ export class TreeSitterExtractor {
|
||||
private fnRefCandidates: Array<FnRefCandidate & { fromNodeId: string }> = [];
|
||||
// Memoized "is this a Vue store file" verdict (per-extractor = per-file).
|
||||
private vueStoreFile: boolean | null = null;
|
||||
// Source already went through the extractor's preParse at the kernel route
|
||||
// point (this instance is the wasm fallback for a kernel-deferred file) —
|
||||
// don't blank it a second time.
|
||||
private sourceIsPreParsed = false;
|
||||
|
||||
constructor(filePath: string, source: string, language?: Language) {
|
||||
constructor(
|
||||
filePath: string,
|
||||
source: string,
|
||||
language?: Language,
|
||||
options?: { sourceIsPreParsed?: boolean }
|
||||
) {
|
||||
this.filePath = filePath;
|
||||
this.source = source;
|
||||
this.language = language || detectLanguage(filePath, source);
|
||||
this.extractor = EXTRACTORS[this.language] || null;
|
||||
this.fnRefSpec = FN_REF_SPECS[this.language];
|
||||
this.sourceIsPreParsed = options?.sourceIsPreParsed === true;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -484,8 +494,9 @@ export class TreeSitterExtractor {
|
||||
// grammar gaps — e.g. C# blanks conditional-compilation directive lines
|
||||
// the grammar mis-parses inside enum bodies (#237). We reassign
|
||||
// this.source so downstream getNodeText reads the same bytes the parser
|
||||
// saw (identical outside the blanked directive lines).
|
||||
if (this.extractor?.preParse) {
|
||||
// saw (identical outside the blanked directive lines). Skipped when the
|
||||
// kernel route point already applied it (sourceIsPreParsed).
|
||||
if (this.extractor?.preParse && !this.sourceIsPreParsed) {
|
||||
this.source = this.extractor.preParse(this.source, this.filePath);
|
||||
}
|
||||
this.tree = parser.parse(this.source) ?? null;
|
||||
@@ -6708,7 +6719,15 @@ export function extractFromSource(
|
||||
if (kernelResult) {
|
||||
result = kernelResult;
|
||||
} else {
|
||||
const extractor = new TreeSitterExtractor(filePath, source, detectedLanguage);
|
||||
// A kernel-deferred file already paid the (offset-preserving) preParse
|
||||
// at the route point — reuse those bytes instead of blanking again.
|
||||
const deferredPre = takeDeferredPreParse(filePath, source, detectedLanguage);
|
||||
const extractor = new TreeSitterExtractor(
|
||||
filePath,
|
||||
deferredPre ?? source,
|
||||
detectedLanguage,
|
||||
{ sourceIsPreParsed: deferredPre != null }
|
||||
);
|
||||
result = extractor.extract();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user