feat(kernel): R1 scaffold — napi-rs extraction kernel, buffer contract, routing + fallback, grammar-parity CI

Phase 0 of the Rust extraction-kernel migration (docs/design/ rust-kernel-migration-plan.md, now checked in with §3a recording the shipped state): - codegraph-kernel/ napi-rs crate: extractFile(path, content, language) → five flat buffers (meta/nodes/edges/refs/arena), one JS boundary crossing per file. Node ids computed Rust-side, byte-identical to generateNodeId (pinned by test vector). Reserved per-node metrics slot for the Arc 3.2 code-metrics work. - Generic .scm-driven emitter (@def.<kind>/@name/@ref.<kind> captures, byte-range scope stack → ::-joined qualified names, contains edges, refs attributed to the innermost enclosing symbol). Seed TS/JS queries are smoke-level; R2 replaces them with the full port. - Routing seam in extractFromSource with per-file wasm fallback. DEFAULT_ROUTED is empty — no behavior change until a language passes its equivalence gate (R3). Dev opt-in: CODEGRAPH_KERNEL_LANGS. Kill switch: CODEGRAPH_KERNEL=0. Loader verifies ABI + kind tables before routing; EDGE_KINDS became a runtime array because kind order is now wire contract. - Grammar-source parity: vendored TS/TSX/JS wasm grammars built from the exact crate revisions (tree-sitter-typescript v0.23.2, tree-sitter-javascript v0.25.0, checked-in parser.c, ts-cli 0.25.10) — the tree-sitter-wasms builds were 2023-era, which the new kernel-grammar-parity test caught on day one. Production TS/JS parsing gets 2.5 years of grammar fixes; full suite green (2456 tests). - Build/release wiring: scripts/build-kernel.sh + npm run build:kernel; release.yml kernel prebuild matrix (continue-on-error — the kernel is optional everywhere, bundles fall back to the wasm path); bundles stage lib/kernel/codegraph-kernel.node; release job runs the kernel suites with CODEGRAPH_KERNEL_EXPECT=1. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-16 20:13:51 -05:00
parent 4efc6c70e2
commit c5eebe6beb
29 changed files with 2804 additions and 16 deletions
@@ -0,0 +1,148 @@
+/**
+ * Native-kernel loader — finds, loads, and contract-verifies the
+ * codegraph-kernel .node addon.
+ *
+ * The kernel is OPTIONAL everywhere. Every failure mode here (no binary for
+ * this platform, dlopen error, ABI/kind-table mismatch) resolves to `null`
+ * and the extraction path silently keeps using the wasm pipeline — a missing
+ * or stale kernel must never break indexing, only skip the speedup. Set
+ * CODEGRAPH_KERNEL_DEBUG=1 to see why a kernel didn't load.
+ *
+ * Kill switch: CODEGRAPH_KERNEL=0 disables the kernel entirely (checked per
+ * call so tests and embedders can flip it at runtime).
+ *
+ * Search order:
+ *   1. CODEGRAPH_KERNEL_PATH — explicit .node path (dev/testing override)
+ *   2. <up3>/kernel/codegraph-kernel.node — the release bundle layout
+ *      (lib/dist/** next to lib/kernel/; see scripts/build-bundle.sh)
+ *   3. <up3>/codegraph-kernel/prebuilds/<platform>-<arch>/codegraph-kernel.node
+ *      — from-source runs and tests (staged by scripts/build-kernel.sh)
+ *
+ * "up3" = three directories above this file, which is the package root both
+ * from src/extraction/kernel/ and from dist/extraction/kernel/.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import { createRequire } from 'module';
+import { NODE_KINDS, EDGE_KINDS } from '../../types';
+import { KERNEL_ABI_VERSION } from './layout';
+
+/** Raw buffer tables for one file — see layout.ts for the byte layout. */
+export interface KernelBuffers {
+  meta: Buffer;
+  nodes: Buffer;
+  edges: Buffer;
+  refs: Buffer;
+  arena: Buffer;
+}
+
+export interface KernelContractInfo {
+  abiVersion: number;
+  kernelVersion: string;
+  nodeKinds: string[];
+  edgeKinds: string[];
+  languages: string[];
+}
+
+export interface KernelGrammarInfo {
+  abiVersion: number;
+  nodeKindCount: number;
+  fieldCount: number;
+  nodeKinds: string[];
+  fieldNames: string[];
+}
+
+export interface KernelModule {
+  extractFile(filePath: string, content: string, language: string): KernelBuffers;
+  contractInfo(): KernelContractInfo;
+  grammarInfo(language: string): KernelGrammarInfo | null;
+}
+
+const debugEnabled = () => process.env.CODEGRAPH_KERNEL_DEBUG === '1';
+function debug(msg: string): void {
+  if (debugEnabled()) process.stderr.write(`[codegraph-kernel] ${msg}\n`);
+}
+
+/** Languages the loaded binary supports (contract-verified). Empty when no kernel. */
+let kernelLanguages: ReadonlySet<string> = new Set();
+/** undefined = not attempted yet; null = attempted and unavailable. */
+let cached: KernelModule | null | undefined;
+
+function candidatePaths(): string[] {
+  const candidates: string[] = [];
+  if (process.env.CODEGRAPH_KERNEL_PATH) candidates.push(process.env.CODEGRAPH_KERNEL_PATH);
+  const packageRoot = path.resolve(__dirname, '..', '..', '..');
+  candidates.push(path.join(packageRoot, 'kernel', 'codegraph-kernel.node'));
+  candidates.push(
+    path.join(
+      packageRoot,
+      'codegraph-kernel',
+      'prebuilds',
+      `${process.platform}-${process.arch}`,
+      'codegraph-kernel.node'
+    )
+  );
+  return candidates;
+}
+
+/**
+ * Verify the binary speaks our wire contract: same ABI version and byte-equal
+ * NodeKind/EdgeKind tables (kinds cross the boundary as indexes into these).
+ */
+function verifyContract(mod: KernelModule, from: string): boolean {
+  const info = mod.contractInfo();
+  if (info.abiVersion !== KERNEL_ABI_VERSION) {
+    debug(`${from}: ABI ${info.abiVersion} != expected ${KERNEL_ABI_VERSION} — ignoring kernel`);
+    return false;
+  }
+  const sameTable = (a: readonly string[], b: readonly string[]) =>
+    a.length === b.length && a.every((v, i) => v === b[i]);
+  if (!sameTable(info.nodeKinds, NODE_KINDS) || !sameTable(info.edgeKinds, EDGE_KINDS)) {
+    debug(`${from}: NodeKind/EdgeKind tables differ from src/types.ts — ignoring kernel`);
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Load (once per process) and return the kernel module, or null when
+ * unavailable. The kill switch is NOT checked here — callers route through
+ * `kernelAvailable()` / `tryKernelExtract()` which check it per call.
+ */
+export function getKernel(): KernelModule | null {
+  if (cached !== undefined) return cached;
+  cached = null;
+  for (const candidate of candidatePaths()) {
+    try {
+      if (!fs.existsSync(candidate)) continue;
+      // createRequire: works identically from CJS output and future ESM.
+      const req = createRequire(__filename);
+      const mod = req(candidate) as KernelModule;
+      if (typeof mod.extractFile !== 'function' || typeof mod.contractInfo !== 'function') {
+        debug(`${candidate}: missing expected exports — ignoring`);
+        continue;
+      }
+      if (!verifyContract(mod, candidate)) continue;
+      kernelLanguages = new Set(mod.contractInfo().languages);
+      debug(`loaded ${candidate} (languages: ${[...kernelLanguages].join(', ')})`);
+      cached = mod;
+      break;
+    } catch (err) {
+      debug(`${candidate}: failed to load — ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  return cached;
+}
+
+/** True when the kill switch is off, a verified binary is loaded, and it supports `language`. */
+export function kernelSupports(language: string): boolean {
+  if (process.env.CODEGRAPH_KERNEL === '0') return false;
+  return getKernel() !== null && kernelLanguages.has(language);
+}
+
+/** Test hook: forget the loaded module so a changed env is re-evaluated. */
+export function resetKernelForTests(): void {
+  cached = undefined;
+  kernelLanguages = new Set();
+}