feat(kernel): R2 — full TypeScript/JavaScript extraction port, byte-parity with the wasm path
Replaces the R1 seed .scm emitter with a bespoke Rust walker (codegraph-kernel/src/tsjs/) that mirrors TreeSitterExtractor's TS/JS paths function-for-function: declarations (incl. #808 field/property classification), qualified names, docstrings (#780 wrapper climbs), signatures, imports/re-exports + per-binding refs, calls with receiver-qualified callees (#1230 literal-receiver skip), instantiations, decorators, inheritance, type annotations (#381), type-alias members + tuple contracts (#359/#634), React component recognition (#841 forwardRef/memo/styled), object-of-functions / zustand-through-middleware / RTK Query endpoints + generated hooks / vuex + pinia store shapes, function-as-value capture with the flush gate (#756), and value-reference edges with the shadow prune (#895/#897). The generic query emitter is deleted — extraction parity needs logic .scm can't express; future languages get walkers too (migration plan §4a). Positions and JS string-slice semantics are emitted in UTF-16 code units natively, so kernel output is byte-identical to web-tree-sitter's — no column diff class exists. Parity evidence (macOS): scripts/kernel-parity.mjs (full-object multiset diff per file) — this repo 353/353 files, excalidraw 643/643 (10,650 nodes / 10,726 edges / 68,307 refs), plus torture fixtures checked into __tests__/fixtures/kernel-parity/ and enforced in npm test by kernel-tsjs-parity.test.ts. The strict compare caught one real decoder bug the loose harness missed: refs must NOT carry denormalized filePath/language at the extractFromSource seam (the store fills them). Perf: extraction 2.6× single-thread on excalidraw (487ms vs 1,255ms, identical outputs). Routing stays opt-in (CODEGRAPH_KERNEL_LANGS) until the R3 equivalence gate (large repo, DB dump-diff, retrieval invariants, agent A/B, Linux/Windows) passes. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
co-authored by
Claude Fable 5
Claude Fable 5
parent
c5eebe6beb
commit
9ad5cd7ba2
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Kernel↔wasm extraction parity harness (R2/R3 of the kernel migration).
|
||||
*
|
||||
* Runs BOTH extraction paths over the given files/directories and diffs the
|
||||
* per-file ExtractionResults as sets (nodes/edges/refs, canonicalized), so a
|
||||
* behavioral gap in the native kernel shows up as a categorized diff instead
|
||||
* of a graph-dump surprise later. This is the fast inner loop; the §5 gate's
|
||||
* full-repo dump-diff still runs before any default-on.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/kernel-parity.mjs <file-or-dir>... [--lang typescript,tsx]
|
||||
* [--max-samples N] [--list-files]
|
||||
*
|
||||
* Requires: npm run build (dist/) and a staged kernel (npm run build:kernel).
|
||||
* Exit code: 0 = parity, 1 = diffs found, 2 = setup error.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const dist = (p) => path.join(ROOT, 'dist', p);
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const paths = [];
|
||||
let langFilter = null;
|
||||
let maxSamples = 5;
|
||||
let listFiles = false;
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--lang') langFilter = new Set(args[++i].split(','));
|
||||
else if (args[i] === '--max-samples') maxSamples = Number(args[++i]);
|
||||
else if (args[i] === '--list-files') listFiles = true;
|
||||
else paths.push(args[i]);
|
||||
}
|
||||
if (paths.length === 0) {
|
||||
console.error('usage: kernel-parity.mjs <file-or-dir>... [--lang ts,tsx] [--max-samples N]');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const KERNEL_LANGS = new Set(['typescript', 'tsx', 'javascript', 'jsx']);
|
||||
const EXTS = new Map([
|
||||
['.ts', 'typescript'], ['.mts', 'typescript'], ['.cts', 'typescript'],
|
||||
['.tsx', 'tsx'], ['.js', 'javascript'], ['.mjs', 'javascript'],
|
||||
['.cjs', 'javascript'], ['.jsx', 'jsx'],
|
||||
]);
|
||||
|
||||
/** Collect candidate files. */
|
||||
function collect(p, out) {
|
||||
const st = fs.statSync(p);
|
||||
if (st.isDirectory()) {
|
||||
const base = path.basename(p);
|
||||
if (base === 'node_modules' || base === '.git' || base === 'dist' || base === '.codegraph') return;
|
||||
for (const e of fs.readdirSync(p)) collect(path.join(p, e), out);
|
||||
} else if (EXTS.has(path.extname(p))) {
|
||||
const lang = EXTS.get(path.extname(p));
|
||||
if (!langFilter || langFilter.has(lang)) out.push({ file: p, lang });
|
||||
}
|
||||
}
|
||||
|
||||
const files = [];
|
||||
for (const p of paths) collect(path.resolve(p), files);
|
||||
if (files.length === 0) {
|
||||
console.error('no matching files');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// --- load the built engine ---------------------------------------------------
|
||||
const { extractFromSource } = await import(dist('extraction/tree-sitter.js'));
|
||||
const { initGrammars, loadGrammarsForLanguages } = await import(dist('extraction/grammars.js'));
|
||||
const kernel = await import(dist('extraction/kernel/index.js'));
|
||||
|
||||
await initGrammars();
|
||||
await loadGrammarsForLanguages([...KERNEL_LANGS]);
|
||||
|
||||
if (!kernel.getKernel()) {
|
||||
console.error('kernel .node not found — run: npm run build:kernel');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// --- canonicalization ---------------------------------------------------------
|
||||
/**
|
||||
* Node identity for cross-referencing edges/refs: the node id itself (both
|
||||
* paths compute the same deterministic ids, and id embeds kind+name+line).
|
||||
*/
|
||||
function canonNode(n) {
|
||||
const out = {
|
||||
id: n.id, kind: n.kind, name: n.name, qualifiedName: n.qualifiedName,
|
||||
filePath: n.filePath, language: n.language,
|
||||
startLine: n.startLine, endLine: n.endLine,
|
||||
startColumn: n.startColumn, endColumn: n.endColumn,
|
||||
};
|
||||
for (const k of ['docstring', 'signature', 'visibility', 'isExported', 'isAsync', 'isStatic', 'isAbstract', 'returnType']) {
|
||||
if (n[k] !== undefined) out[k] = n[k];
|
||||
}
|
||||
if (n.decorators !== undefined) out.decorators = n.decorators;
|
||||
if (n.typeParameters !== undefined) out.typeParameters = n.typeParameters;
|
||||
return JSON.stringify(out);
|
||||
}
|
||||
|
||||
function canonEdge(e) {
|
||||
const out = { source: e.source, target: e.target, kind: e.kind };
|
||||
if (e.line !== undefined) out.line = e.line;
|
||||
if (e.column !== undefined) out.column = e.column;
|
||||
if (e.provenance !== undefined) out.provenance = e.provenance;
|
||||
if (e.metadata !== undefined) out.metadata = e.metadata;
|
||||
return JSON.stringify(out);
|
||||
}
|
||||
|
||||
function canonRef(r) {
|
||||
// FULL object — a field only one path sets is a parity bug (the vitest
|
||||
// parity suite caught decode.ts pre-filling filePath/language this way).
|
||||
const out = {
|
||||
from: r.fromNodeId, name: r.referenceName, kind: r.referenceKind,
|
||||
line: r.line, column: r.column,
|
||||
};
|
||||
for (const k of ['filePath', 'language', 'candidates', 'rowId']) {
|
||||
if (r[k] !== undefined) out[k] = r[k];
|
||||
}
|
||||
return JSON.stringify(out);
|
||||
}
|
||||
|
||||
function diffSets(aList, bList) {
|
||||
const a = new Map(); // canon -> count (multiset — duplicates matter)
|
||||
const b = new Map();
|
||||
for (const x of aList) a.set(x, (a.get(x) ?? 0) + 1);
|
||||
for (const x of bList) b.set(x, (b.get(x) ?? 0) + 1);
|
||||
const onlyA = [];
|
||||
const onlyB = [];
|
||||
for (const [k, c] of a) {
|
||||
const d = c - (b.get(k) ?? 0);
|
||||
for (let i = 0; i < d; i++) onlyA.push(k);
|
||||
}
|
||||
for (const [k, c] of b) {
|
||||
const d = c - (a.get(k) ?? 0);
|
||||
for (let i = 0; i < d; i++) onlyB.push(k);
|
||||
}
|
||||
return { onlyA, onlyB };
|
||||
}
|
||||
|
||||
// --- run ----------------------------------------------------------------------
|
||||
const buckets = new Map(); // category -> {count, samples[]}
|
||||
function report(category, sample) {
|
||||
let b = buckets.get(category);
|
||||
if (!b) buckets.set(category, (b = { count: 0, samples: [] }));
|
||||
b.count++;
|
||||
if (b.samples.length < maxSamples) b.samples.push(sample);
|
||||
}
|
||||
|
||||
let filesWithDiffs = 0;
|
||||
let filesOk = 0;
|
||||
let kernelFailed = 0;
|
||||
let totals = { nodes: 0, edges: 0, refs: 0 };
|
||||
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'all';
|
||||
|
||||
for (const { file, lang } of files) {
|
||||
const source = fs.readFileSync(file, 'utf8');
|
||||
const rel = path.relative(ROOT, file);
|
||||
|
||||
delete process.env.CODEGRAPH_KERNEL; // kernel path on
|
||||
const kres = kernel.tryKernelExtract(rel, source, lang);
|
||||
if (!kres) {
|
||||
kernelFailed++;
|
||||
report('kernel-extract-failed', rel);
|
||||
continue;
|
||||
}
|
||||
process.env.CODEGRAPH_KERNEL = '0'; // wasm path
|
||||
const wres = extractFromSource(rel, source, lang);
|
||||
delete process.env.CODEGRAPH_KERNEL;
|
||||
|
||||
totals.nodes += wres.nodes.length;
|
||||
totals.edges += wres.edges.length;
|
||||
totals.refs += wres.unresolvedReferences.length;
|
||||
|
||||
let fileHasDiff = false;
|
||||
const tables = [
|
||||
['node', wres.nodes.map(canonNode), kres.nodes.map(canonNode)],
|
||||
['edge', wres.edges.map(canonEdge), kres.edges.map(canonEdge)],
|
||||
['ref', wres.unresolvedReferences.map(canonRef), kres.unresolvedReferences.map(canonRef)],
|
||||
];
|
||||
for (const [table, wasm, kern] of tables) {
|
||||
const { onlyA, onlyB } = diffSets(wasm, kern);
|
||||
for (const x of onlyA) {
|
||||
fileHasDiff = true;
|
||||
const o = JSON.parse(x);
|
||||
report(`${table}:missing-in-kernel:${o.kind ?? ''}`, `${rel}: ${x}`);
|
||||
}
|
||||
for (const x of onlyB) {
|
||||
fileHasDiff = true;
|
||||
const o = JSON.parse(x);
|
||||
report(`${table}:extra-in-kernel:${o.kind ?? ''}`, `${rel}: ${x}`);
|
||||
}
|
||||
}
|
||||
if (fileHasDiff) {
|
||||
filesWithDiffs++;
|
||||
if (listFiles) console.log(`DIFF ${rel}`);
|
||||
} else {
|
||||
filesOk++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n=== kernel parity: ${filesOk}/${files.length} files byte-parity` +
|
||||
` (${filesWithDiffs} with diffs, ${kernelFailed} kernel-failed)` +
|
||||
` | wasm totals: ${totals.nodes} nodes / ${totals.edges} edges / ${totals.refs} refs ===\n`);
|
||||
|
||||
const sorted = [...buckets.entries()].sort((a, b) => b[1].count - a[1].count);
|
||||
for (const [cat, { count, samples }] of sorted) {
|
||||
console.log(`--- ${cat}: ${count}`);
|
||||
for (const s of samples) console.log(` ${s.length > 400 ? s.slice(0, 400) + '…' : s}`);
|
||||
}
|
||||
|
||||
process.exit(filesWithDiffs > 0 || kernelFailed > 0 ? 1 : 0);
|
||||
Reference in New Issue
Block a user