feat(kernel): R1 scaffold — napi-rs extraction kernel, buffer contract, routing + fallback, grammar-parity CI
Phase 0 of the Rust extraction-kernel migration (docs/design/ rust-kernel-migration-plan.md, now checked in with §3a recording the shipped state): - codegraph-kernel/ napi-rs crate: extractFile(path, content, language) → five flat buffers (meta/nodes/edges/refs/arena), one JS boundary crossing per file. Node ids computed Rust-side, byte-identical to generateNodeId (pinned by test vector). Reserved per-node metrics slot for the Arc 3.2 code-metrics work. - Generic .scm-driven emitter (@def.<kind>/@name/@ref.<kind> captures, byte-range scope stack → ::-joined qualified names, contains edges, refs attributed to the innermost enclosing symbol). Seed TS/JS queries are smoke-level; R2 replaces them with the full port. - Routing seam in extractFromSource with per-file wasm fallback. DEFAULT_ROUTED is empty — no behavior change until a language passes its equivalence gate (R3). Dev opt-in: CODEGRAPH_KERNEL_LANGS. Kill switch: CODEGRAPH_KERNEL=0. Loader verifies ABI + kind tables before routing; EDGE_KINDS became a runtime array because kind order is now wire contract. - Grammar-source parity: vendored TS/TSX/JS wasm grammars built from the exact crate revisions (tree-sitter-typescript v0.23.2, tree-sitter-javascript v0.25.0, checked-in parser.c, ts-cli 0.25.10) — the tree-sitter-wasms builds were 2023-era, which the new kernel-grammar-parity test caught on day one. Production TS/JS parsing gets 2.5 years of grammar fixes; full suite green (2456 tests). - Build/release wiring: scripts/build-kernel.sh + npm run build:kernel; release.yml kernel prebuild matrix (continue-on-error — the kernel is optional everywhere, bundles fall back to the wasm path); bundles stage lib/kernel/codegraph-kernel.node; release job runs the kernel suites with CODEGRAPH_KERNEL_EXPECT=1. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
4efc6c70e2
commit
c5eebe6beb
@@ -271,10 +271,23 @@ export async function initGrammars(): Promise<void> {
|
||||
* nix-community/tree-sitter-nix @ 3d0173d (MIT) with tree-sitter-cli 0.25.10
|
||||
* (`generate` + `build --wasm`, ABI 15 — upstream's checked-in parser.c is
|
||||
* still ABI 13; all 54 upstream corpus tests pass on the regenerated parser).
|
||||
*
|
||||
* TypeScript/TSX/JavaScript (+jsx, which shares the javascript grammar): the
|
||||
* tree-sitter-wasms builds are 2023-era (^0.20.x); we vendor wasm built from
|
||||
* the SAME grammar revisions the native extraction kernel compiles
|
||||
* (codegraph-kernel/Cargo.toml), so the kernel path and the wasm fallback
|
||||
* parse identically and per-language routing stays graph-neutral:
|
||||
* - tree-sitter/tree-sitter-typescript v0.23.2 (f975a62) → typescript + tsx
|
||||
* - tree-sitter/tree-sitter-javascript v0.25.0 (44c892e) → javascript + jsx
|
||||
* Built from each repo's CHECKED-IN parser.c (no `generate`) with
|
||||
* tree-sitter-cli 0.25.10 `build --wasm` — the same tables crates.io compiles.
|
||||
* The kernel-grammar-parity test asserts this alignment; bump the crate and
|
||||
* the vendored wasm together.
|
||||
*/
|
||||
const VENDORED_WASM_LANGS: ReadonlySet<GrammarLanguage> = new Set([
|
||||
'pascal', 'scala', 'lua', 'luau', 'csharp', 'r', 'cfml', 'cfscript', 'cfquery',
|
||||
'cobol', 'vbnet', 'erlang', 'terraform', 'arkts', 'nix',
|
||||
'typescript', 'tsx', 'javascript', 'jsx',
|
||||
]);
|
||||
|
||||
/** Absolute path of a language's grammar WASM (vendored or tree-sitter-wasms). */
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Decode the kernel's flat buffers into an ExtractionResult — the single
|
||||
* JS-side pass over the per-file tables. See layout.ts for the byte layout
|
||||
* and codegraph-kernel/src/buffers.rs for the writer.
|
||||
*/
|
||||
|
||||
import type {
|
||||
Edge,
|
||||
EdgeKind,
|
||||
ExtractionError,
|
||||
ExtractionResult,
|
||||
Language,
|
||||
Node,
|
||||
NodeKind,
|
||||
ReferenceKind,
|
||||
UnresolvedReference,
|
||||
} from '../../types';
|
||||
import { NODE_KINDS, EDGE_KINDS } from '../../types';
|
||||
import type { KernelBuffers } from './loader';
|
||||
import {
|
||||
EDGE,
|
||||
EDGE_ROW_SIZE,
|
||||
FLAG,
|
||||
FUNCTION_REF_CODE,
|
||||
KERNEL_ABI_VERSION,
|
||||
META,
|
||||
META_SIZE,
|
||||
NODE,
|
||||
NODE_ROW_SIZE,
|
||||
NONE,
|
||||
PROVENANCES,
|
||||
REF,
|
||||
REF_ROW_SIZE,
|
||||
VISIBILITIES,
|
||||
} from './layout';
|
||||
|
||||
/** Read an (offset, len) arena string; undefined when absent. */
|
||||
function str(arena: Buffer, row: Buffer, at: number): string | undefined {
|
||||
const off = row.readUInt32LE(at);
|
||||
if (off === NONE) return undefined;
|
||||
const len = row.readUInt32LE(at + 4);
|
||||
return arena.toString('utf8', off, off + len);
|
||||
}
|
||||
|
||||
/** NUL-joined list field; undefined when absent. */
|
||||
function strList(arena: Buffer, row: Buffer, at: number): string[] | undefined {
|
||||
const joined = str(arena, row, at);
|
||||
return joined === undefined ? undefined : joined.split('\0');
|
||||
}
|
||||
|
||||
/** Tri-state boolean from a (present, value) bit pair. */
|
||||
function flag(flags: number, pair: number): boolean | undefined {
|
||||
if ((flags & (1 << (pair * 2))) === 0) return undefined;
|
||||
return (flags & (1 << (pair * 2 + 1))) !== 0;
|
||||
}
|
||||
|
||||
function u32opt(row: Buffer, at: number): number | undefined {
|
||||
const v = row.readUInt32LE(at);
|
||||
return v === NONE ? undefined : v;
|
||||
}
|
||||
|
||||
export function decodeExtractBuffers(
|
||||
buffers: KernelBuffers,
|
||||
filePath: string,
|
||||
language: Language
|
||||
): ExtractionResult {
|
||||
const { meta, arena } = buffers;
|
||||
if (meta.length < META_SIZE) throw new Error(`kernel meta too short: ${meta.length}`);
|
||||
const version = meta.readUInt8(META.version);
|
||||
if (version !== KERNEL_ABI_VERSION) {
|
||||
throw new Error(`kernel buffer ABI ${version} != expected ${KERNEL_ABI_VERSION}`);
|
||||
}
|
||||
const nodeCount = meta.readUInt32LE(META.nodeCount);
|
||||
const edgeCount = meta.readUInt32LE(META.edgeCount);
|
||||
const refCount = meta.readUInt32LE(META.refCount);
|
||||
|
||||
const now = Date.now();
|
||||
const nodes: Node[] = new Array(nodeCount);
|
||||
// Node-table row index → node id, for edge/ref endpoint resolution.
|
||||
const idByRow: string[] = new Array(nodeCount);
|
||||
|
||||
for (let i = 0; i < nodeCount; i++) {
|
||||
const row = buffers.nodes.subarray(i * NODE_ROW_SIZE, (i + 1) * NODE_ROW_SIZE);
|
||||
const id = str(arena, row, NODE.id)!;
|
||||
idByRow[i] = id;
|
||||
const flags = row.readUInt16LE(NODE.flags);
|
||||
const node: Node = {
|
||||
id,
|
||||
kind: NODE_KINDS[row.readUInt8(NODE.kind)] as NodeKind,
|
||||
name: str(arena, row, NODE.name)!,
|
||||
qualifiedName: str(arena, row, NODE.qualifiedName)!,
|
||||
filePath,
|
||||
language,
|
||||
startLine: row.readUInt32LE(NODE.startLine),
|
||||
endLine: row.readUInt32LE(NODE.endLine),
|
||||
startColumn: row.readUInt32LE(NODE.startColumn),
|
||||
endColumn: row.readUInt32LE(NODE.endColumn),
|
||||
updatedAt: now,
|
||||
};
|
||||
const docstring = str(arena, row, NODE.docstring);
|
||||
if (docstring !== undefined) node.docstring = docstring;
|
||||
const signature = str(arena, row, NODE.signature);
|
||||
if (signature !== undefined) node.signature = signature;
|
||||
const visibility = VISIBILITIES[row.readUInt8(NODE.visibility)];
|
||||
if (visibility !== undefined) node.visibility = visibility;
|
||||
const isExported = flag(flags, FLAG.isExported);
|
||||
if (isExported !== undefined) node.isExported = isExported;
|
||||
const isAsync = flag(flags, FLAG.isAsync);
|
||||
if (isAsync !== undefined) node.isAsync = isAsync;
|
||||
const isStatic = flag(flags, FLAG.isStatic);
|
||||
if (isStatic !== undefined) node.isStatic = isStatic;
|
||||
const isAbstract = flag(flags, FLAG.isAbstract);
|
||||
if (isAbstract !== undefined) node.isAbstract = isAbstract;
|
||||
const decorators = strList(arena, row, NODE.decorators);
|
||||
if (decorators !== undefined) node.decorators = decorators;
|
||||
const typeParameters = strList(arena, row, NODE.typeParameters);
|
||||
if (typeParameters !== undefined) node.typeParameters = typeParameters;
|
||||
const returnType = str(arena, row, NODE.returnType);
|
||||
if (returnType !== undefined) node.returnType = returnType;
|
||||
const extraJson = str(arena, row, NODE.extraJson);
|
||||
if (extraJson !== undefined) Object.assign(node, JSON.parse(extraJson) as Partial<Node>);
|
||||
nodes[i] = node;
|
||||
}
|
||||
|
||||
const edges: Edge[] = new Array(edgeCount);
|
||||
for (let i = 0; i < edgeCount; i++) {
|
||||
const row = buffers.edges.subarray(i * EDGE_ROW_SIZE, (i + 1) * EDGE_ROW_SIZE);
|
||||
const sourceIdx = row.readUInt32LE(EDGE.sourceIdx);
|
||||
const targetIdx = row.readUInt32LE(EDGE.targetIdx);
|
||||
const edge: Edge = {
|
||||
source: sourceIdx === NONE ? str(arena, row, EDGE.sourceIdStr)! : idByRow[sourceIdx]!,
|
||||
target: targetIdx === NONE ? str(arena, row, EDGE.targetIdStr)! : idByRow[targetIdx]!,
|
||||
kind: EDGE_KINDS[row.readUInt8(EDGE.kind)] as EdgeKind,
|
||||
};
|
||||
const line = u32opt(row, EDGE.line);
|
||||
if (line !== undefined) edge.line = line;
|
||||
const column = u32opt(row, EDGE.column);
|
||||
if (column !== undefined) edge.column = column;
|
||||
const provenance = PROVENANCES[row.readUInt8(EDGE.provenance)];
|
||||
if (provenance !== undefined) edge.provenance = provenance;
|
||||
const metadataJson = str(arena, row, EDGE.metadataJson);
|
||||
if (metadataJson !== undefined) edge.metadata = JSON.parse(metadataJson) as Record<string, unknown>;
|
||||
edges[i] = edge;
|
||||
}
|
||||
|
||||
const unresolvedReferences: UnresolvedReference[] = new Array(refCount);
|
||||
for (let i = 0; i < refCount; i++) {
|
||||
const row = buffers.refs.subarray(i * REF_ROW_SIZE, (i + 1) * REF_ROW_SIZE);
|
||||
const fromIdx = row.readUInt32LE(REF.fromIdx);
|
||||
const kindByte = row.readUInt8(REF.kind);
|
||||
const ref: UnresolvedReference = {
|
||||
fromNodeId: fromIdx === NONE ? str(arena, row, REF.fromIdStr)! : idByRow[fromIdx]!,
|
||||
referenceName: str(arena, row, REF.referenceName)!,
|
||||
referenceKind:
|
||||
kindByte === FUNCTION_REF_CODE
|
||||
? 'function_ref'
|
||||
: (EDGE_KINDS[kindByte] as ReferenceKind),
|
||||
line: row.readUInt32LE(REF.line),
|
||||
column: row.readUInt32LE(REF.column),
|
||||
filePath,
|
||||
language,
|
||||
};
|
||||
const candidates = strList(arena, row, REF.candidates);
|
||||
if (candidates !== undefined) ref.candidates = candidates;
|
||||
unresolvedReferences[i] = ref;
|
||||
}
|
||||
|
||||
let errors: ExtractionError[] = [];
|
||||
const errorsOff = meta.readUInt32LE(META.errorsOff);
|
||||
if (errorsOff !== NONE) {
|
||||
const errorsLen = meta.readUInt32LE(META.errorsLen);
|
||||
errors = JSON.parse(arena.toString('utf8', errorsOff, errorsOff + errorsLen)) as ExtractionError[];
|
||||
}
|
||||
|
||||
return { nodes, edges, unresolvedReferences, errors, durationMs: 0 };
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* Kernel routing — which languages go through the native kernel, and the
|
||||
* single entry point the extraction path calls.
|
||||
*
|
||||
* Routing policy is deliberately TS-side and per-language (migration plan §2):
|
||||
* a language routes to the kernel only after its equivalence gate passes;
|
||||
* everything else stays on the wasm path forever if need be. Rollback per
|
||||
* language = removing it from DEFAULT_ROUTED (or CODEGRAPH_KERNEL=0 for all).
|
||||
*
|
||||
* R1 status: NO language is default-routed yet. Development/testing opt-in:
|
||||
* CODEGRAPH_KERNEL_LANGS=typescript,tsx (or "all" for every kernel-capable
|
||||
* language). R3 flips TS/JS into DEFAULT_ROUTED once the gate passes.
|
||||
*/
|
||||
|
||||
import type { ExtractionResult, Language } from '../../types';
|
||||
import { getKernel, kernelSupports } from './loader';
|
||||
import { decodeExtractBuffers } from './decode';
|
||||
|
||||
export { getKernel, kernelSupports, resetKernelForTests } from './loader';
|
||||
export { decodeExtractBuffers } from './decode';
|
||||
|
||||
/**
|
||||
* Languages routed to the kernel by default (gate-passed only — see the
|
||||
* per-language tracker in docs/design/rust-kernel-migration-plan.md §4).
|
||||
*/
|
||||
const DEFAULT_ROUTED: ReadonlySet<Language> = new Set<Language>([]);
|
||||
|
||||
/**
|
||||
* Per-language TS post-pass over the decoded result — the escape hatch for
|
||||
* logic `.scm` queries can't express (macro salvage, dialect sniffing,
|
||||
* wrapper-based component recognition). Runs synchronously after decode,
|
||||
* before the framework extract() hooks the caller applies. Keep these SMALL:
|
||||
* anything heavy belongs in the Rust emitter.
|
||||
*/
|
||||
export type KernelPostPass = (result: ExtractionResult, source: string) => void;
|
||||
const POST_PASSES: Partial<Record<Language, KernelPostPass>> = {
|
||||
// (none yet — R2+)
|
||||
};
|
||||
|
||||
function isRouted(language: Language): boolean {
|
||||
const env = process.env.CODEGRAPH_KERNEL_LANGS;
|
||||
if (env === undefined || env === '') return DEFAULT_ROUTED.has(language);
|
||||
if (env === 'all') return true;
|
||||
return env
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.includes(language);
|
||||
}
|
||||
|
||||
/** True when `language` would be extracted by the kernel right now. */
|
||||
export function kernelRoutes(language: Language): boolean {
|
||||
return isRouted(language) && kernelSupports(language);
|
||||
}
|
||||
|
||||
/** Warned-once registry so a broken language logs a single line, not one per file. */
|
||||
const warned = new Set<string>();
|
||||
|
||||
/**
|
||||
* Extract via the native kernel. Returns null when the kernel doesn't apply
|
||||
* (not routed / not available / kill switch) — the caller falls back to the
|
||||
* wasm TreeSitterExtractor. A kernel ERROR on a routed file also returns
|
||||
* null: per-file fallback keeps indexing correct while a kernel bug costs
|
||||
* only that file's speedup.
|
||||
*/
|
||||
export function tryKernelExtract(
|
||||
filePath: string,
|
||||
source: string,
|
||||
language: Language
|
||||
): ExtractionResult | null {
|
||||
if (!kernelRoutes(language)) return null;
|
||||
const kernel = getKernel();
|
||||
if (!kernel) return null;
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
// NOTE(T2 languages): when a preParse-carrying language (csharp #237,
|
||||
// metal #1121, cuda #1172, c/cpp macro blanking) routes here, its
|
||||
// offset-preserving preParse hook must be applied to `source` first —
|
||||
// wire that alongside the language's port, gated WITH its equivalence run.
|
||||
const buffers = kernel.extractFile(filePath, source, language);
|
||||
const result = decodeExtractBuffers(buffers, filePath, language);
|
||||
POST_PASSES[language]?.(result, source);
|
||||
result.durationMs = Date.now() - t0;
|
||||
return result;
|
||||
} catch (err) {
|
||||
if (!warned.has(language)) {
|
||||
warned.add(language);
|
||||
process.stderr.write(
|
||||
`[codegraph-kernel] ${language} extraction failed (${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}) — falling back to the wasm path\n`
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Native-kernel buffer layout — TS mirror of codegraph-kernel/src/buffers.rs.
|
||||
*
|
||||
* The kernel returns five Buffers per file: meta, nodes, edges, refs, arena.
|
||||
* Rows are fixed-width little-endian; strings are (offset, len) pairs into
|
||||
* the UTF-8 arena; `offset === NONE` means "field absent".
|
||||
*
|
||||
* THIS FILE AND buffers.rs MUST MATCH BYTE FOR BYTE. Any layout change bumps
|
||||
* KERNEL_ABI_VERSION on both sides — the loader refuses a version it doesn't
|
||||
* know and the extraction path falls back to wasm.
|
||||
*
|
||||
* NodeKind / EdgeKind / provenance / visibility cross the boundary as indexes
|
||||
* into NODE_KINDS / EDGE_KINDS (src/types.ts) and the small tables below, so
|
||||
* those array orders are part of the contract (append, never reorder). The
|
||||
* loader additionally verifies the kernel's own kind tables against
|
||||
* NODE_KINDS/EDGE_KINDS at load time, so a stale .node degrades to the wasm
|
||||
* path instead of mis-decoding.
|
||||
*/
|
||||
|
||||
export const KERNEL_ABI_VERSION = 1;
|
||||
|
||||
/** Sentinel for "absent" in u32 slots and string-ref offsets. */
|
||||
export const NONE = 0xffffffff;
|
||||
|
||||
export const META_SIZE = 36;
|
||||
export const NODE_ROW_SIZE = 96;
|
||||
export const EDGE_ROW_SIZE = 44;
|
||||
export const REF_ROW_SIZE = 40;
|
||||
|
||||
/** meta byte offsets */
|
||||
export const META = {
|
||||
version: 0, // u8
|
||||
nodeCount: 4, // u32
|
||||
edgeCount: 8, // u32
|
||||
refCount: 12, // u32
|
||||
arenaLen: 16, // u32
|
||||
errorsOff: 20, // u32 (NONE = no errors)
|
||||
errorsLen: 24, // u32
|
||||
durationMs: 28, // f64 (kernel-side wall; introspection only)
|
||||
} as const;
|
||||
|
||||
/** node row byte offsets */
|
||||
export const NODE = {
|
||||
kind: 0, // u8 — NODE_KINDS index
|
||||
visibility: 1, // u8 — VISIBILITIES index (0 = absent)
|
||||
flags: 2, // u16 — (present, value) bit pairs, see FLAG
|
||||
startLine: 4, // u32
|
||||
endLine: 8, // u32
|
||||
startColumn: 12, // u32
|
||||
endColumn: 16, // u32
|
||||
name: 20, // str
|
||||
qualifiedName: 28, // str
|
||||
id: 36, // str — kernel-computed node id
|
||||
docstring: 44, // str
|
||||
signature: 52, // str
|
||||
decorators: 60, // str — NUL-joined list
|
||||
typeParameters: 68, // str — NUL-joined list
|
||||
returnType: 76, // str
|
||||
extraJson: 84, // str — JSON of any extra Node props (escape hatch)
|
||||
metrics: 92, // u32 — reserved (Arc 3.2 per-node code metrics)
|
||||
} as const;
|
||||
|
||||
/** edge row byte offsets */
|
||||
export const EDGE = {
|
||||
sourceIdx: 0, // u32 (NONE → sourceIdStr)
|
||||
targetIdx: 4, // u32 (NONE → targetIdStr)
|
||||
kind: 8, // u8 — EDGE_KINDS index
|
||||
provenance: 9, // u8 — PROVENANCES index (0 = absent)
|
||||
line: 12, // u32 (NONE = absent)
|
||||
column: 16, // u32 (NONE = absent)
|
||||
metadataJson: 20, // str
|
||||
sourceIdStr: 28, // str
|
||||
targetIdStr: 36, // str
|
||||
} as const;
|
||||
|
||||
/** ref row byte offsets */
|
||||
export const REF = {
|
||||
fromIdx: 0, // u32 (NONE → fromIdStr)
|
||||
kind: 4, // u8 — EDGE_KINDS index, or FUNCTION_REF_CODE
|
||||
line: 8, // u32
|
||||
column: 12, // u32
|
||||
referenceName: 16, // str
|
||||
candidates: 24, // str — NUL-joined list
|
||||
fromIdStr: 32, // str
|
||||
} as const;
|
||||
|
||||
/** ReferenceKind wire code for the internal-only `function_ref` (#756). */
|
||||
export const FUNCTION_REF_CODE = 200;
|
||||
|
||||
/** Node bool-flag bit pairs: bit(2n) = present, bit(2n+1) = value. */
|
||||
export const FLAG = {
|
||||
isExported: 0,
|
||||
isAsync: 1,
|
||||
isStatic: 2,
|
||||
isAbstract: 3,
|
||||
} as const;
|
||||
|
||||
/** visibility byte values (0 = absent). */
|
||||
export const VISIBILITIES = [undefined, 'public', 'private', 'protected', 'internal'] as const;
|
||||
|
||||
/** provenance byte values (0 = absent). */
|
||||
export const PROVENANCES = [undefined, 'tree-sitter', 'scip', 'heuristic'] as const;
|
||||
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* Native-kernel loader — finds, loads, and contract-verifies the
|
||||
* codegraph-kernel .node addon.
|
||||
*
|
||||
* The kernel is OPTIONAL everywhere. Every failure mode here (no binary for
|
||||
* this platform, dlopen error, ABI/kind-table mismatch) resolves to `null`
|
||||
* and the extraction path silently keeps using the wasm pipeline — a missing
|
||||
* or stale kernel must never break indexing, only skip the speedup. Set
|
||||
* CODEGRAPH_KERNEL_DEBUG=1 to see why a kernel didn't load.
|
||||
*
|
||||
* Kill switch: CODEGRAPH_KERNEL=0 disables the kernel entirely (checked per
|
||||
* call so tests and embedders can flip it at runtime).
|
||||
*
|
||||
* Search order:
|
||||
* 1. CODEGRAPH_KERNEL_PATH — explicit .node path (dev/testing override)
|
||||
* 2. <up3>/kernel/codegraph-kernel.node — the release bundle layout
|
||||
* (lib/dist/** next to lib/kernel/; see scripts/build-bundle.sh)
|
||||
* 3. <up3>/codegraph-kernel/prebuilds/<platform>-<arch>/codegraph-kernel.node
|
||||
* — from-source runs and tests (staged by scripts/build-kernel.sh)
|
||||
*
|
||||
* "up3" = three directories above this file, which is the package root both
|
||||
* from src/extraction/kernel/ and from dist/extraction/kernel/.
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { createRequire } from 'module';
|
||||
import { NODE_KINDS, EDGE_KINDS } from '../../types';
|
||||
import { KERNEL_ABI_VERSION } from './layout';
|
||||
|
||||
/** Raw buffer tables for one file — see layout.ts for the byte layout. */
|
||||
export interface KernelBuffers {
|
||||
meta: Buffer;
|
||||
nodes: Buffer;
|
||||
edges: Buffer;
|
||||
refs: Buffer;
|
||||
arena: Buffer;
|
||||
}
|
||||
|
||||
export interface KernelContractInfo {
|
||||
abiVersion: number;
|
||||
kernelVersion: string;
|
||||
nodeKinds: string[];
|
||||
edgeKinds: string[];
|
||||
languages: string[];
|
||||
}
|
||||
|
||||
export interface KernelGrammarInfo {
|
||||
abiVersion: number;
|
||||
nodeKindCount: number;
|
||||
fieldCount: number;
|
||||
nodeKinds: string[];
|
||||
fieldNames: string[];
|
||||
}
|
||||
|
||||
export interface KernelModule {
|
||||
extractFile(filePath: string, content: string, language: string): KernelBuffers;
|
||||
contractInfo(): KernelContractInfo;
|
||||
grammarInfo(language: string): KernelGrammarInfo | null;
|
||||
}
|
||||
|
||||
const debugEnabled = () => process.env.CODEGRAPH_KERNEL_DEBUG === '1';
|
||||
function debug(msg: string): void {
|
||||
if (debugEnabled()) process.stderr.write(`[codegraph-kernel] ${msg}\n`);
|
||||
}
|
||||
|
||||
/** Languages the loaded binary supports (contract-verified). Empty when no kernel. */
|
||||
let kernelLanguages: ReadonlySet<string> = new Set();
|
||||
/** undefined = not attempted yet; null = attempted and unavailable. */
|
||||
let cached: KernelModule | null | undefined;
|
||||
|
||||
function candidatePaths(): string[] {
|
||||
const candidates: string[] = [];
|
||||
if (process.env.CODEGRAPH_KERNEL_PATH) candidates.push(process.env.CODEGRAPH_KERNEL_PATH);
|
||||
const packageRoot = path.resolve(__dirname, '..', '..', '..');
|
||||
candidates.push(path.join(packageRoot, 'kernel', 'codegraph-kernel.node'));
|
||||
candidates.push(
|
||||
path.join(
|
||||
packageRoot,
|
||||
'codegraph-kernel',
|
||||
'prebuilds',
|
||||
`${process.platform}-${process.arch}`,
|
||||
'codegraph-kernel.node'
|
||||
)
|
||||
);
|
||||
return candidates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify the binary speaks our wire contract: same ABI version and byte-equal
|
||||
* NodeKind/EdgeKind tables (kinds cross the boundary as indexes into these).
|
||||
*/
|
||||
function verifyContract(mod: KernelModule, from: string): boolean {
|
||||
const info = mod.contractInfo();
|
||||
if (info.abiVersion !== KERNEL_ABI_VERSION) {
|
||||
debug(`${from}: ABI ${info.abiVersion} != expected ${KERNEL_ABI_VERSION} — ignoring kernel`);
|
||||
return false;
|
||||
}
|
||||
const sameTable = (a: readonly string[], b: readonly string[]) =>
|
||||
a.length === b.length && a.every((v, i) => v === b[i]);
|
||||
if (!sameTable(info.nodeKinds, NODE_KINDS) || !sameTable(info.edgeKinds, EDGE_KINDS)) {
|
||||
debug(`${from}: NodeKind/EdgeKind tables differ from src/types.ts — ignoring kernel`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load (once per process) and return the kernel module, or null when
|
||||
* unavailable. The kill switch is NOT checked here — callers route through
|
||||
* `kernelAvailable()` / `tryKernelExtract()` which check it per call.
|
||||
*/
|
||||
export function getKernel(): KernelModule | null {
|
||||
if (cached !== undefined) return cached;
|
||||
cached = null;
|
||||
for (const candidate of candidatePaths()) {
|
||||
try {
|
||||
if (!fs.existsSync(candidate)) continue;
|
||||
// createRequire: works identically from CJS output and future ESM.
|
||||
const req = createRequire(__filename);
|
||||
const mod = req(candidate) as KernelModule;
|
||||
if (typeof mod.extractFile !== 'function' || typeof mod.contractInfo !== 'function') {
|
||||
debug(`${candidate}: missing expected exports — ignoring`);
|
||||
continue;
|
||||
}
|
||||
if (!verifyContract(mod, candidate)) continue;
|
||||
kernelLanguages = new Set(mod.contractInfo().languages);
|
||||
debug(`loaded ${candidate} (languages: ${[...kernelLanguages].join(', ')})`);
|
||||
cached = mod;
|
||||
break;
|
||||
} catch (err) {
|
||||
debug(`${candidate}: failed to load — ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
|
||||
/** True when the kill switch is off, a verified binary is loaded, and it supports `language`. */
|
||||
export function kernelSupports(language: string): boolean {
|
||||
if (process.env.CODEGRAPH_KERNEL === '0') return false;
|
||||
return getKernel() !== null && kernelLanguages.has(language);
|
||||
}
|
||||
|
||||
/** Test hook: forget the loaded module so a changed env is re-evaluated. */
|
||||
export function resetKernelForTests(): void {
|
||||
cached = undefined;
|
||||
kernelLanguages = new Set();
|
||||
}
|
||||
@@ -30,6 +30,7 @@ import { DfmExtractor } from './dfm-extractor';
|
||||
import { VueExtractor } from './vue-extractor';
|
||||
import { MyBatisExtractor } from './mybatis-extractor';
|
||||
import { CfmlExtractor } from './cfml-extractor';
|
||||
import { tryKernelExtract } from './kernel';
|
||||
import {
|
||||
getAllFrameworkResolvers,
|
||||
getApplicableFrameworks,
|
||||
@@ -6700,8 +6701,16 @@ export function extractFromSource(
|
||||
const extractor = new DfmExtractor(filePath, source);
|
||||
result = extractor.extract();
|
||||
} else {
|
||||
const extractor = new TreeSitterExtractor(filePath, source, detectedLanguage);
|
||||
result = extractor.extract();
|
||||
// Native-kernel route (docs/design/rust-kernel-migration-plan.md): gated
|
||||
// per language, null when not routed/available or on a kernel error —
|
||||
// the wasm TreeSitterExtractor below stays the fallback either way.
|
||||
const kernelResult = tryKernelExtract(filePath, source, detectedLanguage);
|
||||
if (kernelResult) {
|
||||
result = kernelResult;
|
||||
} else {
|
||||
const extractor = new TreeSitterExtractor(filePath, source, detectedLanguage);
|
||||
result = extractor.extract();
|
||||
}
|
||||
}
|
||||
|
||||
// Framework-specific extraction (routes, middleware, etc.)
|
||||
|
||||
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
BIN
Binary file not shown.
Reference in New Issue
Block a user