feat(kernel): R1 scaffold — napi-rs extraction kernel, buffer contract, routing + fallback, grammar-parity CI
Phase 0 of the Rust extraction-kernel migration (docs/design/ rust-kernel-migration-plan.md, now checked in with §3a recording the shipped state): - codegraph-kernel/ napi-rs crate: extractFile(path, content, language) → five flat buffers (meta/nodes/edges/refs/arena), one JS boundary crossing per file. Node ids computed Rust-side, byte-identical to generateNodeId (pinned by test vector). Reserved per-node metrics slot for the Arc 3.2 code-metrics work. - Generic .scm-driven emitter (@def.<kind>/@name/@ref.<kind> captures, byte-range scope stack → ::-joined qualified names, contains edges, refs attributed to the innermost enclosing symbol). Seed TS/JS queries are smoke-level; R2 replaces them with the full port. - Routing seam in extractFromSource with per-file wasm fallback. DEFAULT_ROUTED is empty — no behavior change until a language passes its equivalence gate (R3). Dev opt-in: CODEGRAPH_KERNEL_LANGS. Kill switch: CODEGRAPH_KERNEL=0. Loader verifies ABI + kind tables before routing; EDGE_KINDS became a runtime array because kind order is now wire contract. - Grammar-source parity: vendored TS/TSX/JS wasm grammars built from the exact crate revisions (tree-sitter-typescript v0.23.2, tree-sitter-javascript v0.25.0, checked-in parser.c, ts-cli 0.25.10) — the tree-sitter-wasms builds were 2023-era, which the new kernel-grammar-parity test caught on day one. Production TS/JS parsing gets 2.5 years of grammar fixes; full suite green (2456 tests). - Build/release wiring: scripts/build-kernel.sh + npm run build:kernel; release.yml kernel prebuild matrix (continue-on-error — the kernel is optional everywhere, bundles fall back to the wasm path); bundles stage lib/kernel/codegraph-kernel.node; release job runs the kernel suites with CODEGRAPH_KERNEL_EXPECT=1. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
co-authored by
Claude Fable 5
Claude Fable 5
parent
4efc6c70e2
commit
c5eebe6beb
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* Native-kernel scaffold tests (R1, docs/design/rust-kernel-migration-plan.md).
|
||||
*
|
||||
* Covers the wire contract, decoder, routing policy, kill switch, and
|
||||
* per-file fallback. These are SCAFFOLD tests — behavioral parity with the
|
||||
* wasm extractors is R3's equivalence gate, not asserted here.
|
||||
*
|
||||
* The kernel binary is optional: without a staged .node
|
||||
* (scripts/build-kernel.sh) the suite skips. CI that builds the kernel sets
|
||||
* CODEGRAPH_KERNEL_EXPECT=1, which turns "missing binary" into a FAILURE so
|
||||
* the gate can't silently pass by not building the kernel.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { NODE_KINDS, EDGE_KINDS } from '../src/types';
|
||||
import { generateNodeId } from '../src/extraction/tree-sitter-helpers';
|
||||
import { getKernel, tryKernelExtract, kernelRoutes, resetKernelForTests } from '../src/extraction/kernel';
|
||||
import { extractFromSource } from '../src/extraction';
|
||||
import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars';
|
||||
|
||||
const KERNEL_PATH = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'codegraph-kernel',
|
||||
'prebuilds',
|
||||
`${process.platform}-${process.arch}`,
|
||||
'codegraph-kernel.node'
|
||||
);
|
||||
const kernelBuilt = fs.existsSync(KERNEL_PATH);
|
||||
const expectKernel = process.env.CODEGRAPH_KERNEL_EXPECT === '1';
|
||||
|
||||
const FIXTURE = [
|
||||
'export class MathHelper {',
|
||||
' calculateTotal(a: number): number { return helper(a); }',
|
||||
'}',
|
||||
'function helper(x: number): number { return x * 2; }',
|
||||
'helper(3);',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS', 'CODEGRAPH_KERNEL_PATH'] as const;
|
||||
let savedEnv: Record<string, string | undefined>;
|
||||
|
||||
beforeEach(() => {
|
||||
savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
|
||||
for (const k of ENV_KEYS) delete process.env[k];
|
||||
resetKernelForTests();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const k of ENV_KEYS) {
|
||||
if (savedEnv[k] === undefined) delete process.env[k];
|
||||
else process.env[k] = savedEnv[k];
|
||||
}
|
||||
resetKernelForTests();
|
||||
});
|
||||
|
||||
it.runIf(expectKernel)('kernel binary must exist when CODEGRAPH_KERNEL_EXPECT=1', () => {
|
||||
expect(kernelBuilt, `expected kernel at ${KERNEL_PATH} — run scripts/build-kernel.sh`).toBe(true);
|
||||
});
|
||||
|
||||
describe.skipIf(!kernelBuilt)('kernel scaffold', () => {
|
||||
it('loads and its kind tables match src/types.ts exactly', () => {
|
||||
const kernel = getKernel();
|
||||
expect(kernel).not.toBeNull();
|
||||
const info = kernel!.contractInfo();
|
||||
expect(info.nodeKinds).toEqual([...NODE_KINDS]);
|
||||
expect(info.edgeKinds).toEqual([...EDGE_KINDS]);
|
||||
expect(info.languages).toContain('typescript');
|
||||
expect(info.languages).toContain('javascript');
|
||||
});
|
||||
|
||||
it('no language routes to the kernel by default (R1: wasm path unchanged)', () => {
|
||||
expect(kernelRoutes('typescript')).toBe(false);
|
||||
expect(tryKernelExtract('src/a.ts', 'function f() {}', 'typescript')).toBeNull();
|
||||
});
|
||||
|
||||
describe('with typescript routed (CODEGRAPH_KERNEL_LANGS)', () => {
|
||||
beforeEach(() => {
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'typescript';
|
||||
});
|
||||
|
||||
it('decodes nodes, contains edges, and calls refs from the buffers', () => {
|
||||
const result = tryKernelExtract('src/utils.ts', FIXTURE, 'typescript');
|
||||
expect(result).not.toBeNull();
|
||||
const { nodes, edges, unresolvedReferences, errors } = result!;
|
||||
expect(errors).toEqual([]);
|
||||
|
||||
const byKind = (kind: string) => nodes.filter((n) => n.kind === kind);
|
||||
expect(byKind('file')).toHaveLength(1);
|
||||
expect(byKind('class').map((n) => n.name)).toEqual(['MathHelper']);
|
||||
expect(byKind('method').map((n) => n.qualifiedName)).toEqual(['MathHelper::calculateTotal']);
|
||||
expect(byKind('function').map((n) => n.name)).toEqual(['helper']);
|
||||
|
||||
const file = byKind('file')[0]!;
|
||||
expect(file.id).toBe('file:src/utils.ts');
|
||||
expect(file.qualifiedName).toBe('src/utils.ts');
|
||||
expect(file.endLine).toBe(FIXTURE.split('\n').length);
|
||||
expect(file.isExported).toBe(false);
|
||||
|
||||
// Every node carries the decode-call constants.
|
||||
for (const n of nodes) {
|
||||
expect(n.filePath).toBe('src/utils.ts');
|
||||
expect(n.language).toBe('typescript');
|
||||
expect(n.updatedAt).toBeGreaterThan(0);
|
||||
}
|
||||
|
||||
// contains: file→class, class→method, file→function.
|
||||
const contains = edges.filter((e) => e.kind === 'contains');
|
||||
const cls = byKind('class')[0]!;
|
||||
const method = byKind('method')[0]!;
|
||||
const fn = byKind('function')[0]!;
|
||||
expect(contains).toContainEqual({ source: file.id, target: cls.id, kind: 'contains' });
|
||||
expect(contains).toContainEqual({ source: cls.id, target: method.id, kind: 'contains' });
|
||||
expect(contains).toContainEqual({ source: file.id, target: fn.id, kind: 'contains' });
|
||||
|
||||
// calls refs attach to the innermost enclosing symbol (method for the
|
||||
// in-body call, file node for the top-level call).
|
||||
const calls = unresolvedReferences.filter((r) => r.referenceKind === 'calls');
|
||||
expect(calls.map((r) => [r.fromNodeId, r.referenceName])).toEqual([
|
||||
[method.id, 'helper'],
|
||||
[file.id, 'helper'],
|
||||
]);
|
||||
for (const r of calls) {
|
||||
expect(r.filePath).toBe('src/utils.ts');
|
||||
expect(r.language).toBe('typescript');
|
||||
expect(r.line).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('kernel node ids are byte-identical to generateNodeId', () => {
|
||||
const result = tryKernelExtract('src/utils.ts', FIXTURE, 'typescript')!;
|
||||
for (const n of result.nodes) {
|
||||
if (n.kind === 'file') continue;
|
||||
expect(n.id).toBe(generateNodeId('src/utils.ts', n.kind, n.name, n.startLine));
|
||||
}
|
||||
});
|
||||
|
||||
it('CODEGRAPH_KERNEL=0 kill switch disables routing', () => {
|
||||
process.env.CODEGRAPH_KERNEL = '0';
|
||||
expect(kernelRoutes('typescript')).toBe(false);
|
||||
expect(tryKernelExtract('src/a.ts', FIXTURE, 'typescript')).toBeNull();
|
||||
});
|
||||
|
||||
it('languages outside the route stay on the wasm path', () => {
|
||||
expect(kernelRoutes('javascript')).toBe(false);
|
||||
expect(tryKernelExtract('src/a.js', 'function f() {}', 'javascript')).toBeNull();
|
||||
});
|
||||
|
||||
it('tsx routes with its own entry and returns a graph', () => {
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'typescript,tsx';
|
||||
const result = tryKernelExtract(
|
||||
'src/App.tsx',
|
||||
'export function App() { return render(); }\n',
|
||||
'tsx'
|
||||
);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.nodes.some((n) => n.kind === 'function' && n.name === 'App')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractFromSource seam', () => {
|
||||
beforeAll(async () => {
|
||||
await initGrammars();
|
||||
await loadGrammarsForLanguages(['typescript']);
|
||||
});
|
||||
|
||||
it('unrouted language flows through the wasm extractor unchanged', () => {
|
||||
// `const f = () => 1` yields a function node on the wasm path; the seed
|
||||
// kernel query deliberately doesn't extract it — so its presence proves
|
||||
// which path ran.
|
||||
const result = extractFromSource('src/a.ts', 'export const f = () => 1;\n', 'typescript');
|
||||
expect(result.nodes.some((n) => n.kind === 'function' && n.name === 'f')).toBe(true);
|
||||
});
|
||||
|
||||
it('routed language takes the kernel and falls back per file on kernel absence', () => {
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'typescript';
|
||||
const viaKernel = extractFromSource('src/utils.ts', FIXTURE, 'typescript');
|
||||
expect(viaKernel.nodes.map((n) => n.kind)).toContain('method');
|
||||
|
||||
// Point the loader at a nonexistent binary: routing is requested but the
|
||||
// kernel can't load, so the SAME call must fall back to wasm, not fail.
|
||||
process.env.CODEGRAPH_KERNEL_PATH = path.join(__dirname, 'nope', 'missing.node');
|
||||
process.env.CODEGRAPH_KERNEL = '0'; // and belt-and-braces the kill switch
|
||||
resetKernelForTests();
|
||||
const viaWasm = extractFromSource('src/utils.ts', FIXTURE, 'typescript');
|
||||
expect(viaWasm.nodes.some((n) => n.kind === 'class' && n.name === 'MathHelper')).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user