feat(extraction): C deferral round 2 — 8 new preParse passes, linux kernel/+mm/ deferral 58.6%→33.9% (#1353)
Census-driven cut of the top-ranked post-R7a lever. All passes TS-side, C-only (preParseCSource), shared by both arms: - parameterized-annotation whole-blank (__free/__printf/__counted_by/ __bpf_md_ptr…; extends through a stranded field `;`) - type-keyword-arg scanner (kzalloc_obj(struct T), list_entry, multi-line continuations behind nested-paren args; bounded hand scanner, head exclusions + call-vs-declaration guard; blanks trailing stars) - static/extern CAPS-macro declaration lines at any scope; the initialized form is REWRITTEN to its expansion (name/tail keep exact offsets) - va_arg qualified-type blank; GNU named-variadic #define dots-only blank (post-restore); sandwiched notrace-family; C23 auto; multi-line iterator-macro spans (hlist_for_each_entry_rcu + lockdep arg) - word list += cacheline family (2- and 4-underscore spellings) + 10 more census-confirmed annotations Gates: five-repo parity sweeps 0 diffs (git deferral 16.1→12.2%, redis 25.3→24.1%, fmt/protobuf unchanged); linux full-tree both arms 2,049,153 nodes / 6,413,518 edges (+858/+6,585 vs R7a) with byte-identical dumps (10,446,478 lines, sha256 6dd1185b); kernel-arm parse-loop 356→306s at 2c; suite 2517 green under CODEGRAPH_KERNEL_EXPECT=1. Honesty note recorded in the docs: error recovery was already salvaging most SYMBOLS on deferred files — the graph win is relationships + phantom cleanup, and the unreleased CHANGELOG entry was rewritten off the sweep-subset framing. Also records §7a.5: post-R7a 8-core cg1212 re-run 16.4min (was 18.3min); 8c parse sits on the single-writer floor, so the <10min-on-8c gap re-ranks to the per-ref resolution path. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby Mchenry
co-authored by
Claude Fable 5
Claude Fable 5
parent
2d72891b59
commit
b9d0f57a64
@@ -11280,6 +11280,171 @@ describe('C/C++ kernel-port preParse blanks (R7a)', () => {
|
||||
expect(out).toContain('__u32 count');
|
||||
});
|
||||
|
||||
it('blankCParameterizedAnnotationMacros blanks name+args whole, eats a stranded field semicolon', async () => {
|
||||
const { blankCParameterizedAnnotationMacros } = await import('../src/extraction/languages/c-cpp');
|
||||
const src = [
|
||||
'struct file *f __free(fput) = NULL;',
|
||||
'static void __printf(4, 0) log_it(int a, const char *fmt, ...);',
|
||||
'struct ctx {',
|
||||
'\t__bpf_md_ptr(struct bpf_iter_meta *, meta);',
|
||||
'};',
|
||||
'int keep = __hash(key);',
|
||||
'',
|
||||
].join('\n');
|
||||
const out = blankCParameterizedAnnotationMacros(src);
|
||||
expect(out.length).toBe(src.length);
|
||||
expect(out).not.toContain('__free');
|
||||
expect(out).not.toContain('__printf');
|
||||
// Mid-line match keeps its statement tail…
|
||||
expect(out).toContain('= NULL;');
|
||||
// …but a whole-line FIELD match eats the `;` too — a lone `;` field is
|
||||
// itself a parse error while an empty struct body is not.
|
||||
expect(out).not.toContain('__bpf_md_ptr');
|
||||
expect(out.split('\n')[3]?.trim()).toBe('');
|
||||
// Non-curated dunder calls are real code.
|
||||
expect(out).toContain('__hash(key)');
|
||||
});
|
||||
|
||||
it('blankCTypeKeywordArgs blanks bare type-keyword call args, spares valid look-alikes', async () => {
|
||||
const { blankCTypeKeywordArgs } = await import('../src/extraction/languages/c-cpp');
|
||||
const src = [
|
||||
'void j(void *head, void *map) {',
|
||||
'\tvoid *p = kzalloc_obj(struct bpf_mount_opts);',
|
||||
'\tvoid *e = list_first_entry(head,',
|
||||
'\t\t\tstruct async_entry, domain_list);',
|
||||
'\tvoid *n = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(head)),',
|
||||
'\t\t\tstruct bpf_dtab_netdev, index_hlist);',
|
||||
'\treturn container_of(map, struct bpf_map, inner);',
|
||||
'}',
|
||||
'DEFINE_PER_CPU(struct task_struct *, ksoftirqd);',
|
||||
'',
|
||||
].join('\n');
|
||||
const out = blankCTypeKeywordArgs(src);
|
||||
expect(out.length).toBe(src.length);
|
||||
expect(out).not.toContain('struct bpf_mount_opts');
|
||||
expect(out).toContain(' bpf_mount_opts'); // keyword → spaces, ident stays
|
||||
expect(out).toContain(' async_entry, domain_list');
|
||||
expect(out).toContain(' bpf_dtab_netdev'); // nested-paren predecessor arg
|
||||
expect(out).toContain(' bpf_map, inner'); // `return` precedes real calls
|
||||
// Pointer form blanks the stars too — two plain identifier args remain.
|
||||
expect(out).toContain(' task_struct , ksoftirqd');
|
||||
// Valid look-alikes stay byte-identical.
|
||||
for (const valid of [
|
||||
'int a = sizeof(struct point);',
|
||||
'int b = offsetof(struct point, y);',
|
||||
'int c = _Generic(x, struct foo *: 1, default: 0);',
|
||||
'int wf(struct a,\n\tstruct b);',
|
||||
'void cast(void *p) { use((struct foo *)p); }',
|
||||
'struct ops { int (*probe)(struct device *dev); };',
|
||||
]) {
|
||||
expect(blankCTypeKeywordArgs(valid)).toBe(valid);
|
||||
}
|
||||
});
|
||||
|
||||
it('blankCFileScopePrefixedDeclMacros blanks static/extern CAPS-macro lines at any scope', async () => {
|
||||
const { blankCFileScopePrefixedDeclMacros } = await import('../src/extraction/languages/c-cpp');
|
||||
const src = [
|
||||
'static DEFINE_PER_CPU(struct llist_head, rstat_backlog_list);',
|
||||
'void f(void) {',
|
||||
'\tstatic DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5);',
|
||||
'}',
|
||||
'EXPORT_SYMBOL(vmalloc);',
|
||||
'static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {',
|
||||
'',
|
||||
].join('\n');
|
||||
const out = blankCFileScopePrefixedDeclMacros(src);
|
||||
expect(out.length).toBe(src.length);
|
||||
expect(out).not.toContain('DEFINE_PER_CPU(struct llist_head');
|
||||
expect(out).not.toContain('DEFINE_RATELIMIT_STATE'); // block scope blanks too
|
||||
// Bare CAPS lines parse natively as K&R declarations — untouched.
|
||||
expect(out).toContain('EXPORT_SYMBOL(vmalloc);');
|
||||
// Initializer forms belong to the rewrite, not the blank.
|
||||
expect(out).toContain('cpuhp_state) = {');
|
||||
});
|
||||
|
||||
it('rewriteCPrefixedDeclMacroInitializers rewrites `static CAPS(type, name) = {` into the declaration', async () => {
|
||||
const { rewriteCPrefixedDeclMacroInitializers } = await import('../src/extraction/languages/c-cpp');
|
||||
const line = 'static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {';
|
||||
const src = [line, '\t.fail = CPUHP_INVALID,', '};', ''].join('\n');
|
||||
const out = rewriteCPrefixedDeclMacroInitializers(src);
|
||||
expect(out.length).toBe(src.length);
|
||||
const rewritten = out.split('\n')[0] as string;
|
||||
expect(rewritten).toContain('static struct cpuhp_cpu_state');
|
||||
expect(rewritten).not.toContain('DEFINE_PER_CPU');
|
||||
// The NAME keeps its exact original column, and the tail its offsets.
|
||||
expect(rewritten.indexOf('cpuhp_state')).not.toBe(-1);
|
||||
expect(rewritten.indexOf('cpuhp_state', 30)).toBe(line.indexOf('cpuhp_state', 30));
|
||||
expect(rewritten.indexOf('= {')).toBe(line.indexOf('= {'));
|
||||
// Three-argument macros never match.
|
||||
const threeArg = 'static DEFINE_TIMER(t, fn, 0) = {\n};\n';
|
||||
expect(rewriteCPrefixedDeclMacroInitializers(threeArg)).toBe(threeArg);
|
||||
});
|
||||
|
||||
it('blankCVaArgQualifiedTypeArgs blanks multi-token va_arg types, spares single tokens', async () => {
|
||||
const { blankCVaArgQualifiedTypeArgs } = await import('../src/extraction/languages/c-cpp');
|
||||
const src = 'void f(va_list ap) {\n\tconst char *s = va_arg(ap, const char *);\n\tint n = va_arg(ap, int);\n}\n';
|
||||
const out = blankCVaArgQualifiedTypeArgs(src);
|
||||
expect(out.length).toBe(src.length);
|
||||
expect(out).toContain('va_arg(ap );');
|
||||
expect(out).toContain('va_arg(ap, int);'); // parses natively — untouched
|
||||
});
|
||||
|
||||
it('blankCNamedVariadicDefineDots blanks only the dots of GNU named-variadic params', async () => {
|
||||
const { blankCNamedVariadicDefineDots } = await import('../src/extraction/languages/c-cpp');
|
||||
const named = '#define verbose(env, fmt, args...) log_write(env, fmt, ##args)\nint x;\n';
|
||||
const out = blankCNamedVariadicDefineDots(named);
|
||||
expect(out.length).toBe(named.length);
|
||||
expect(out).toContain('args )'); // dots → spaces
|
||||
expect(out).toContain('##args'); // body untouched
|
||||
const std = '#define pr(fmt, ...) printk(fmt, __VA_ARGS__)\nint y;\n';
|
||||
expect(blankCNamedVariadicDefineDots(std)).toBe(std);
|
||||
});
|
||||
|
||||
it('blankCSandwichedAnnotations and blankCAutoInference: sandwich and C23-auto guards', async () => {
|
||||
const { blankCSandwichedAnnotations, blankCAutoInference } = await import(
|
||||
'../src/extraction/languages/c-cpp'
|
||||
);
|
||||
const src = 'static notrace void tick_do(void) { }\nstatic nokprobe_inline void arm(void) { }\n';
|
||||
const out = blankCSandwichedAnnotations(src);
|
||||
expect(out.length).toBe(src.length);
|
||||
expect(out).not.toContain('notrace');
|
||||
expect(out).not.toContain('nokprobe_inline');
|
||||
// As a variable name (no following word) it survives.
|
||||
const varUse = 'void f(void) { int notrace = 1; use(notrace); }\n';
|
||||
expect(blankCSandwichedAnnotations(varUse)).toBe(varUse);
|
||||
const c23 = 'void q(void) { auto hb = get_hb(); }\n';
|
||||
const autoOut = blankCAutoInference(c23);
|
||||
expect(autoOut.length).toBe(c23.length);
|
||||
expect(autoOut).toContain(' hb = get_hb();');
|
||||
// The storage-class reading has a TYPE after `auto` — untouched.
|
||||
const storage = 'void s(void) { auto int x = 1; }\n';
|
||||
expect(blankCAutoInference(storage)).toBe(storage);
|
||||
});
|
||||
|
||||
it('blankCStatementMacroCalls spans wrapped iterator macros, spares wrapped real calls', async () => {
|
||||
const { blankCStatementMacroCalls } = await import('../src/extraction/languages/c-cpp');
|
||||
const src = [
|
||||
'static void walk(void *head) {',
|
||||
'\thlist_for_each_entry_rcu(p, head, hlist,',
|
||||
'\t\t\t\t lockdep_is_held(&kprobe_mutex)) {',
|
||||
'\t\tuse(p);',
|
||||
'\t}',
|
||||
'}',
|
||||
'',
|
||||
].join('\n');
|
||||
const out = blankCStatementMacroCalls(src);
|
||||
expect(out.length).toBe(src.length);
|
||||
expect(out).not.toContain('hlist_for_each_entry_rcu');
|
||||
expect(out).not.toContain('lockdep_is_held');
|
||||
expect(out).toContain('use(p);');
|
||||
// A wrapped REAL call ends in `;` — untouched.
|
||||
const call = 'void f(void) {\n\tdo_thing(a,\n\t\t b);\n}\n';
|
||||
expect(blankCStatementMacroCalls(call)).toBe(call);
|
||||
// A wrapped condition is keyword-led — untouched.
|
||||
const cond = 'void g(int a) {\n\tif (check(a,\n\t\t a)) {\n\t\tuse(a);\n\t}\n}\n';
|
||||
expect(blankCStatementMacroCalls(cond)).toBe(cond);
|
||||
});
|
||||
|
||||
it('restoreDirectiveLines keeps #define lines out of the blanking blast radius', async () => {
|
||||
const { extractFromSource } = await import('../src/extraction');
|
||||
// FMT_API matches the _API-suffix member blank; without the directive
|
||||
|
||||
@@ -82,3 +82,73 @@ static void spawn_workers(void) {
|
||||
register_handler(cb_a);
|
||||
signal_connect(&cb_b);
|
||||
}
|
||||
|
||||
/* ---- deferral round 2 (the linux-idiom preParse family) --------------------
|
||||
* Every shape below used to drop the WHOLE file into error recovery (and
|
||||
* therefore defer it to wasm). Each now parses via a round-2 blank/rewrite;
|
||||
* this section pins kernel-vs-wasm parity on all of them at once. */
|
||||
|
||||
/* file-scope prefixed declaration macros — whole-line blank */
|
||||
static DEFINE_PER_CPU(struct llist_head, rstat_backlog_list);
|
||||
static DECLARE_WORK(init_free_wq, do_free_init);
|
||||
extern DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
|
||||
|
||||
/* initialized per-cpu declaration — the REWRITE (type + name survive) */
|
||||
static DEFINE_PER_CPU(struct conn, cpuhp_state) = {
|
||||
.fd = 1,
|
||||
};
|
||||
|
||||
/* type-keyword arguments — keyword (and trailing stars) blank */
|
||||
static void type_args(void *head, void *map) {
|
||||
void *opts = kzalloc_obj(struct conn);
|
||||
void *entry = list_first_entry(head,
|
||||
struct conn, fd);
|
||||
void *outer = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(head)),
|
||||
struct conn, fd);
|
||||
use_ptr(outer, container_of(map, struct conn, fd));
|
||||
use_ptr(opts, entry);
|
||||
}
|
||||
DEFINE_PER_CPU(struct conn *, ksoftirqd);
|
||||
|
||||
/* parameterized annotations — name+args blank whole */
|
||||
static void cleanup_scope(void) {
|
||||
struct conn *token __free(kfree) = NULL;
|
||||
use_ptr(token, token);
|
||||
}
|
||||
static void __printf(1, 2) log_fmt(const char *fmt, ...);
|
||||
struct flex_tail {
|
||||
int count;
|
||||
int owners[] __counted_by(count);
|
||||
};
|
||||
|
||||
/* sandwiched lowercase annotations + C23 auto */
|
||||
static notrace void tick_do(int x) { use_val(x); }
|
||||
static nokprobe_inline void arm_probe(void) { }
|
||||
static void auto_user(void) {
|
||||
auto hb = shadowed_reader();
|
||||
use_val(hb);
|
||||
}
|
||||
|
||||
/* va_arg with a qualified type argument */
|
||||
static void drain_args(va_list ap) {
|
||||
const char *s = va_arg(ap, const char *);
|
||||
int n = va_arg(ap, int);
|
||||
use_ptr((void *)s, (void *)(long)n);
|
||||
}
|
||||
|
||||
/* multi-line statement-position iterator macro */
|
||||
static void walk_rcu(void *head) {
|
||||
hlist_for_each_entry_rcu(pos, head, hlist,
|
||||
lockdep_is_held(&probe_mutex)) {
|
||||
use_ptr(pos, head);
|
||||
}
|
||||
}
|
||||
|
||||
/* GNU named-variadic define — dots blank, body survives */
|
||||
#define verbose(env, fmt, args...) log_writer(env, fmt, ##args)
|
||||
|
||||
/* block-scope prefixed declaration macro */
|
||||
static void ratelimited_warn(void) {
|
||||
static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5);
|
||||
use_ptr(&ratelimit, 0);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user