feat(kernel): R1 scaffold — napi-rs extraction kernel, buffer contract, routing + fallback, grammar-parity CI
Phase 0 of the Rust extraction-kernel migration (docs/design/ rust-kernel-migration-plan.md, now checked in with §3a recording the shipped state): - codegraph-kernel/ napi-rs crate: extractFile(path, content, language) → five flat buffers (meta/nodes/edges/refs/arena), one JS boundary crossing per file. Node ids computed Rust-side, byte-identical to generateNodeId (pinned by test vector). Reserved per-node metrics slot for the Arc 3.2 code-metrics work. - Generic .scm-driven emitter (@def.<kind>/@name/@ref.<kind> captures, byte-range scope stack → ::-joined qualified names, contains edges, refs attributed to the innermost enclosing symbol). Seed TS/JS queries are smoke-level; R2 replaces them with the full port. - Routing seam in extractFromSource with per-file wasm fallback. DEFAULT_ROUTED is empty — no behavior change until a language passes its equivalence gate (R3). Dev opt-in: CODEGRAPH_KERNEL_LANGS. Kill switch: CODEGRAPH_KERNEL=0. Loader verifies ABI + kind tables before routing; EDGE_KINDS became a runtime array because kind order is now wire contract. - Grammar-source parity: vendored TS/TSX/JS wasm grammars built from the exact crate revisions (tree-sitter-typescript v0.23.2, tree-sitter-javascript v0.25.0, checked-in parser.c, ts-cli 0.25.10) — the tree-sitter-wasms builds were 2023-era, which the new kernel-grammar-parity test caught on day one. Production TS/JS parsing gets 2.5 years of grammar fixes; full suite green (2456 tests). - Build/release wiring: scripts/build-kernel.sh + npm run build:kernel; release.yml kernel prebuild matrix (continue-on-error — the kernel is optional everywhere, bundles fall back to the wasm path); bundles stage lib/kernel/codegraph-kernel.node; release job runs the kernel suites with CODEGRAPH_KERNEL_EXPECT=1. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
co-authored by
Claude Fable 5
Claude Fable 5
parent
4efc6c70e2
commit
c5eebe6beb
@@ -29,8 +29,45 @@ permissions:
|
||||
attestations: write # store the GitHub artifact attestations for the bundles
|
||||
|
||||
jobs:
|
||||
# Native extraction-kernel prebuilds (docs/design/rust-kernel-migration-plan.md).
|
||||
# The kernel is an OPTIONAL per-language speedup: a bundle without a .node
|
||||
# runs the wasm extraction path unchanged. continue-on-error keeps a Rust
|
||||
# toolchain flake from ever blocking a release — the release job runs with
|
||||
# whatever prebuilds succeeded. (Runner images ship rustup; build-kernel.sh
|
||||
# adds each cross target itself.)
|
||||
kernel:
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- runner: macos-14
|
||||
targets: aarch64-apple-darwin x86_64-apple-darwin
|
||||
- runner: ubuntu-22.04 # oldest glibc runner → widest compatibility
|
||||
targets: x86_64-unknown-linux-gnu
|
||||
- runner: ubuntu-22.04-arm
|
||||
targets: aarch64-unknown-linux-gnu
|
||||
- runner: windows-latest
|
||||
targets: x86_64-pc-windows-msvc aarch64-pc-windows-msvc
|
||||
runs-on: ${{ matrix.runner }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Build kernel prebuilds
|
||||
shell: bash
|
||||
run: |
|
||||
for t in ${{ matrix.targets }}; do
|
||||
bash scripts/build-kernel.sh --target "$t"
|
||||
done
|
||||
ls -R codegraph-kernel/prebuilds
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: kernel-${{ matrix.runner }}
|
||||
path: codegraph-kernel/prebuilds/
|
||||
if-no-files-found: error
|
||||
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: kernel
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
@@ -127,6 +164,32 @@ jobs:
|
||||
git push origin "HEAD:${GITHUB_REF#refs/heads/}"
|
||||
fi
|
||||
|
||||
- name: Download kernel prebuilds
|
||||
# Best-effort: whatever platform legs succeeded land in release/kernel/
|
||||
# (<target>/codegraph-kernel.node); build-bundle.sh includes a target's
|
||||
# kernel when present and falls back to the wasm path when not.
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: kernel-*
|
||||
merge-multiple: true
|
||||
path: release/kernel/
|
||||
|
||||
- name: Kernel contract + grammar-parity gate
|
||||
# Asserts the native grammars and the vendored wasm grammars are built
|
||||
# from the same grammar revisions (node-kind/field tables compared id
|
||||
# by id) and that the .node speaks the expected wire contract.
|
||||
# CODEGRAPH_KERNEL_EXPECT=1 turns a missing binary into a FAILURE here
|
||||
# so the gate can't silently pass by not building the kernel.
|
||||
run: |
|
||||
if [ -f release/kernel/linux-x64/codegraph-kernel.node ]; then
|
||||
mkdir -p codegraph-kernel/prebuilds/linux-x64
|
||||
cp release/kernel/linux-x64/codegraph-kernel.node codegraph-kernel/prebuilds/linux-x64/
|
||||
CODEGRAPH_KERNEL_EXPECT=1 npx vitest run __tests__/kernel-scaffold.test.ts __tests__/kernel-grammar-parity.test.ts
|
||||
else
|
||||
echo "::warning::no linux-x64 kernel prebuild — skipping kernel gate (bundles ship wasm-only)"
|
||||
fi
|
||||
|
||||
- name: Build all platform bundles
|
||||
run: |
|
||||
for t in darwin-arm64 darwin-x64 linux-x64 linux-arm64 win32-x64 win32-arm64; do
|
||||
|
||||
@@ -22,6 +22,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
### Fixes
|
||||
|
||||
- TypeScript, TSX, and JavaScript files now parse with up-to-date grammars — modern syntax such as `using` declarations and import attributes no longer trips parse errors that could drop surrounding symbols. (The previously bundled grammars dated from 2023.)
|
||||
- Searching or exploring by field names now finds the code that defines them. A query made of object keys or API field names (`profileInfo isTrialEligible quotaInfo billingMethod`) used to return unrelated results while the defining files never appeared, because three retrieval steps each dropped multi-word camelCase terms: an internal case-comparison bug, a match step that only considered classes (never functions or methods), and exploration seeding that required exact symbol-name matches. All three are fixed — `codegraph_explore` with a bag of field names now surfaces the controllers and services that assemble those fields. (#1196)
|
||||
- `codegraph.json`'s `includeIgnored` works again for the "folder of repos" layout: when one `.gitignore` rule covers a parent directory (`/repos/`) holding several embedded git repositories, opting in the individual repos (`"includeIgnored": ["repos/a/"]` — the exact spelling `codegraph init`'s own hint suggests) previously matched nothing and indexed zero files, looping the same suggestion back at you. Both spellings now work — name the parent directory to opt in everything under it, or name individual repos to opt in just those — and the hint no longer re-suggests repos that are already configured. (#1295)
|
||||
- Method calls on literals (`", ".join(...)` in Python, `"x".split(...)` in JavaScript, and the like) no longer produce call edges to unrelated project functions that happen to share the builtin's name — a codebase with a function called `join`, `get`, or `update` could show phantom callers from every string-builtin use. Additionally, a function nested inside another function is now only matched as a call target from inside its container, since it isn't reachable from anywhere else. Blast-radius and affected-test results get cleaner on Python and JavaScript codebases especially. (#1230)
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* Grammar-source parity gate (R1, migration plan §3.5).
|
||||
*
|
||||
* The native kernel compiles grammars from crates.io / vendored sources; the
|
||||
* wasm fallback loads grammars from tree-sitter-wasms / src/extraction/wasm.
|
||||
* If the two are built from different grammar revisions, a language's graph
|
||||
* would depend on WHICH path extracted it — per-language routing (and the
|
||||
* kernel-absent fallback) must be graph-neutral.
|
||||
*
|
||||
* Rather than trusting version metadata, this asserts the grammars are
|
||||
* behaviorally identical where extraction can observe them: ABI version and
|
||||
* the full node-kind and field tables, compared id by id.
|
||||
*
|
||||
* Runs wherever a kernel binary is staged (scripts/build-kernel.sh); skips
|
||||
* otherwise. CI that builds the kernel sets CODEGRAPH_KERNEL_EXPECT=1 so the
|
||||
* skip can't mask a missing build (asserted in kernel-scaffold.test.ts).
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import type { Language as WasmLanguage } from 'web-tree-sitter';
|
||||
import { getKernel, resetKernelForTests } from '../src/extraction/kernel';
|
||||
import { initGrammars, loadGrammarsForLanguages, getParser } from '../src/extraction/grammars';
|
||||
import type { Language } from '../src/types';
|
||||
|
||||
const KERNEL_PATH = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'codegraph-kernel',
|
||||
'prebuilds',
|
||||
`${process.platform}-${process.arch}`,
|
||||
'codegraph-kernel.node'
|
||||
);
|
||||
const kernelBuilt = fs.existsSync(KERNEL_PATH);
|
||||
|
||||
// Every kernel-capable language. `jsx` shares the javascript grammar on BOTH
|
||||
// paths (langs.rs mirrors WASM_GRAMMAR_FILES), so the distinct grammars are:
|
||||
const GRAMMAR_LANGUAGES: Language[] = ['typescript', 'tsx', 'javascript'];
|
||||
|
||||
describe.skipIf(!kernelBuilt)('kernel↔wasm grammar parity', () => {
|
||||
beforeAll(async () => {
|
||||
resetKernelForTests();
|
||||
await initGrammars();
|
||||
await loadGrammarsForLanguages(GRAMMAR_LANGUAGES);
|
||||
});
|
||||
|
||||
it.each(GRAMMAR_LANGUAGES)('%s: node-kind and field tables are identical', (language) => {
|
||||
const kernel = getKernel();
|
||||
expect(kernel).not.toBeNull();
|
||||
const native = kernel!.grammarInfo(language);
|
||||
expect(native, `kernel has no grammar for ${language}`).not.toBeNull();
|
||||
|
||||
const wasmLang = getParser(language)?.language as WasmLanguage | null | undefined;
|
||||
expect(wasmLang, `wasm grammar for ${language} not loaded`).toBeTruthy();
|
||||
|
||||
expect(native!.abiVersion, 'grammar ABI version').toBe(wasmLang!.abiVersion);
|
||||
expect(native!.nodeKindCount, 'node-kind count').toBe(wasmLang!.nodeTypeCount);
|
||||
expect(native!.fieldCount, 'field count').toBe(wasmLang!.fieldCount);
|
||||
|
||||
const wasmKinds: (string | null)[] = [];
|
||||
for (let i = 0; i < wasmLang!.nodeTypeCount; i++) wasmKinds.push(wasmLang!.nodeTypeForId(i));
|
||||
expect(native!.nodeKinds).toEqual(wasmKinds.map((k) => k ?? ''));
|
||||
|
||||
// Field ids are 1-based on both sides.
|
||||
const wasmFields: (string | null)[] = [];
|
||||
for (let i = 1; i <= wasmLang!.fieldCount; i++) wasmFields.push(wasmLang!.fieldNameForId(i));
|
||||
expect(native!.fieldNames).toEqual(wasmFields.map((f) => f ?? ''));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* Native-kernel scaffold tests (R1, docs/design/rust-kernel-migration-plan.md).
|
||||
*
|
||||
* Covers the wire contract, decoder, routing policy, kill switch, and
|
||||
* per-file fallback. These are SCAFFOLD tests — behavioral parity with the
|
||||
* wasm extractors is R3's equivalence gate, not asserted here.
|
||||
*
|
||||
* The kernel binary is optional: without a staged .node
|
||||
* (scripts/build-kernel.sh) the suite skips. CI that builds the kernel sets
|
||||
* CODEGRAPH_KERNEL_EXPECT=1, which turns "missing binary" into a FAILURE so
|
||||
* the gate can't silently pass by not building the kernel.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { NODE_KINDS, EDGE_KINDS } from '../src/types';
|
||||
import { generateNodeId } from '../src/extraction/tree-sitter-helpers';
|
||||
import { getKernel, tryKernelExtract, kernelRoutes, resetKernelForTests } from '../src/extraction/kernel';
|
||||
import { extractFromSource } from '../src/extraction';
|
||||
import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars';
|
||||
|
||||
const KERNEL_PATH = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'codegraph-kernel',
|
||||
'prebuilds',
|
||||
`${process.platform}-${process.arch}`,
|
||||
'codegraph-kernel.node'
|
||||
);
|
||||
const kernelBuilt = fs.existsSync(KERNEL_PATH);
|
||||
const expectKernel = process.env.CODEGRAPH_KERNEL_EXPECT === '1';
|
||||
|
||||
const FIXTURE = [
|
||||
'export class MathHelper {',
|
||||
' calculateTotal(a: number): number { return helper(a); }',
|
||||
'}',
|
||||
'function helper(x: number): number { return x * 2; }',
|
||||
'helper(3);',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS', 'CODEGRAPH_KERNEL_PATH'] as const;
|
||||
let savedEnv: Record<string, string | undefined>;
|
||||
|
||||
beforeEach(() => {
|
||||
savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
|
||||
for (const k of ENV_KEYS) delete process.env[k];
|
||||
resetKernelForTests();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const k of ENV_KEYS) {
|
||||
if (savedEnv[k] === undefined) delete process.env[k];
|
||||
else process.env[k] = savedEnv[k];
|
||||
}
|
||||
resetKernelForTests();
|
||||
});
|
||||
|
||||
it.runIf(expectKernel)('kernel binary must exist when CODEGRAPH_KERNEL_EXPECT=1', () => {
|
||||
expect(kernelBuilt, `expected kernel at ${KERNEL_PATH} — run scripts/build-kernel.sh`).toBe(true);
|
||||
});
|
||||
|
||||
describe.skipIf(!kernelBuilt)('kernel scaffold', () => {
|
||||
it('loads and its kind tables match src/types.ts exactly', () => {
|
||||
const kernel = getKernel();
|
||||
expect(kernel).not.toBeNull();
|
||||
const info = kernel!.contractInfo();
|
||||
expect(info.nodeKinds).toEqual([...NODE_KINDS]);
|
||||
expect(info.edgeKinds).toEqual([...EDGE_KINDS]);
|
||||
expect(info.languages).toContain('typescript');
|
||||
expect(info.languages).toContain('javascript');
|
||||
});
|
||||
|
||||
it('no language routes to the kernel by default (R1: wasm path unchanged)', () => {
|
||||
expect(kernelRoutes('typescript')).toBe(false);
|
||||
expect(tryKernelExtract('src/a.ts', 'function f() {}', 'typescript')).toBeNull();
|
||||
});
|
||||
|
||||
describe('with typescript routed (CODEGRAPH_KERNEL_LANGS)', () => {
|
||||
beforeEach(() => {
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'typescript';
|
||||
});
|
||||
|
||||
it('decodes nodes, contains edges, and calls refs from the buffers', () => {
|
||||
const result = tryKernelExtract('src/utils.ts', FIXTURE, 'typescript');
|
||||
expect(result).not.toBeNull();
|
||||
const { nodes, edges, unresolvedReferences, errors } = result!;
|
||||
expect(errors).toEqual([]);
|
||||
|
||||
const byKind = (kind: string) => nodes.filter((n) => n.kind === kind);
|
||||
expect(byKind('file')).toHaveLength(1);
|
||||
expect(byKind('class').map((n) => n.name)).toEqual(['MathHelper']);
|
||||
expect(byKind('method').map((n) => n.qualifiedName)).toEqual(['MathHelper::calculateTotal']);
|
||||
expect(byKind('function').map((n) => n.name)).toEqual(['helper']);
|
||||
|
||||
const file = byKind('file')[0]!;
|
||||
expect(file.id).toBe('file:src/utils.ts');
|
||||
expect(file.qualifiedName).toBe('src/utils.ts');
|
||||
expect(file.endLine).toBe(FIXTURE.split('\n').length);
|
||||
expect(file.isExported).toBe(false);
|
||||
|
||||
// Every node carries the decode-call constants.
|
||||
for (const n of nodes) {
|
||||
expect(n.filePath).toBe('src/utils.ts');
|
||||
expect(n.language).toBe('typescript');
|
||||
expect(n.updatedAt).toBeGreaterThan(0);
|
||||
}
|
||||
|
||||
// contains: file→class, class→method, file→function.
|
||||
const contains = edges.filter((e) => e.kind === 'contains');
|
||||
const cls = byKind('class')[0]!;
|
||||
const method = byKind('method')[0]!;
|
||||
const fn = byKind('function')[0]!;
|
||||
expect(contains).toContainEqual({ source: file.id, target: cls.id, kind: 'contains' });
|
||||
expect(contains).toContainEqual({ source: cls.id, target: method.id, kind: 'contains' });
|
||||
expect(contains).toContainEqual({ source: file.id, target: fn.id, kind: 'contains' });
|
||||
|
||||
// calls refs attach to the innermost enclosing symbol (method for the
|
||||
// in-body call, file node for the top-level call).
|
||||
const calls = unresolvedReferences.filter((r) => r.referenceKind === 'calls');
|
||||
expect(calls.map((r) => [r.fromNodeId, r.referenceName])).toEqual([
|
||||
[method.id, 'helper'],
|
||||
[file.id, 'helper'],
|
||||
]);
|
||||
for (const r of calls) {
|
||||
expect(r.filePath).toBe('src/utils.ts');
|
||||
expect(r.language).toBe('typescript');
|
||||
expect(r.line).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('kernel node ids are byte-identical to generateNodeId', () => {
|
||||
const result = tryKernelExtract('src/utils.ts', FIXTURE, 'typescript')!;
|
||||
for (const n of result.nodes) {
|
||||
if (n.kind === 'file') continue;
|
||||
expect(n.id).toBe(generateNodeId('src/utils.ts', n.kind, n.name, n.startLine));
|
||||
}
|
||||
});
|
||||
|
||||
it('CODEGRAPH_KERNEL=0 kill switch disables routing', () => {
|
||||
process.env.CODEGRAPH_KERNEL = '0';
|
||||
expect(kernelRoutes('typescript')).toBe(false);
|
||||
expect(tryKernelExtract('src/a.ts', FIXTURE, 'typescript')).toBeNull();
|
||||
});
|
||||
|
||||
it('languages outside the route stay on the wasm path', () => {
|
||||
expect(kernelRoutes('javascript')).toBe(false);
|
||||
expect(tryKernelExtract('src/a.js', 'function f() {}', 'javascript')).toBeNull();
|
||||
});
|
||||
|
||||
it('tsx routes with its own entry and returns a graph', () => {
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'typescript,tsx';
|
||||
const result = tryKernelExtract(
|
||||
'src/App.tsx',
|
||||
'export function App() { return render(); }\n',
|
||||
'tsx'
|
||||
);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.nodes.some((n) => n.kind === 'function' && n.name === 'App')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractFromSource seam', () => {
|
||||
beforeAll(async () => {
|
||||
await initGrammars();
|
||||
await loadGrammarsForLanguages(['typescript']);
|
||||
});
|
||||
|
||||
it('unrouted language flows through the wasm extractor unchanged', () => {
|
||||
// `const f = () => 1` yields a function node on the wasm path; the seed
|
||||
// kernel query deliberately doesn't extract it — so its presence proves
|
||||
// which path ran.
|
||||
const result = extractFromSource('src/a.ts', 'export const f = () => 1;\n', 'typescript');
|
||||
expect(result.nodes.some((n) => n.kind === 'function' && n.name === 'f')).toBe(true);
|
||||
});
|
||||
|
||||
it('routed language takes the kernel and falls back per file on kernel absence', () => {
|
||||
process.env.CODEGRAPH_KERNEL_LANGS = 'typescript';
|
||||
const viaKernel = extractFromSource('src/utils.ts', FIXTURE, 'typescript');
|
||||
expect(viaKernel.nodes.map((n) => n.kind)).toContain('method');
|
||||
|
||||
// Point the loader at a nonexistent binary: routing is requested but the
|
||||
// kernel can't load, so the SAME call must fall back to wasm, not fail.
|
||||
process.env.CODEGRAPH_KERNEL_PATH = path.join(__dirname, 'nope', 'missing.node');
|
||||
process.env.CODEGRAPH_KERNEL = '0'; // and belt-and-braces the kill switch
|
||||
resetKernelForTests();
|
||||
const viaWasm = extractFromSource('src/utils.ts', FIXTURE, 'typescript');
|
||||
expect(viaWasm.nodes.some((n) => n.kind === 'class' && n.name === 'MathHelper')).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,3 @@
|
||||
target/
|
||||
prebuilds/
|
||||
*.node
|
||||
Generated
+542
@@ -0,0 +1,542 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.67"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "codegraph-kernel"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"napi",
|
||||
"napi-build",
|
||||
"napi-derive",
|
||||
"sha2",
|
||||
"streaming-iterator",
|
||||
"tree-sitter",
|
||||
"tree-sitter-javascript",
|
||||
"tree-sitter-typescript",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "convert_case"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49"
|
||||
dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctor"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a394189d59f9befacce833f337f7b1eca5e9a91221bcdd4d28e0114d96e597b3"
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-executor",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-macro",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.186"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "napi"
|
||||
version = "3.10.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6826e5ddc15589b2d68c8ad5321c18e85d40488e93e32962f362e572669bccf6"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"ctor",
|
||||
"futures",
|
||||
"napi-build",
|
||||
"napi-sys",
|
||||
"nohash-hasher",
|
||||
"rustc-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "napi-build"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1"
|
||||
|
||||
[[package]]
|
||||
name = "napi-derive"
|
||||
version = "3.5.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0fe526e81c105d3640516fcde83909dd1afe757c0d7a15af58830b5bc0fb9a1"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"ctor",
|
||||
"napi-derive-backend",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "napi-derive-backend"
|
||||
version = "5.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "514281397bcddd9ea9a876c7a21a57bff2374237a000ca9a64ea0211ec1993e2"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"semver",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "napi-sys"
|
||||
version = "3.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73e43cf2eb0bd1bf95a43c07c076ebd2da5d1e015a71c3d201faeffffcc0ecac"
|
||||
dependencies = [
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nohash-hasher"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.150"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter"
|
||||
version = "0.25.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"regex",
|
||||
"regex-syntax",
|
||||
"serde_json",
|
||||
"streaming-iterator",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-javascript"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-language"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-typescript"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
@@ -0,0 +1,32 @@
|
||||
[package]
|
||||
name = "codegraph-kernel"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "MIT"
|
||||
publish = false
|
||||
description = "Native extraction kernel for CodeGraph — tree-sitter parse+extract with one JS boundary crossing per file"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
napi = { version = "3", default-features = false, features = ["napi8"] }
|
||||
napi-derive = "3"
|
||||
tree-sitter = "0.25"
|
||||
streaming-iterator = "0.1"
|
||||
sha2 = "0.10"
|
||||
|
||||
# Grammars — MUST stay revision-matched with the wasm grammars the fallback
|
||||
# path loads (tree-sitter-wasms npm package / src/extraction/wasm/). The
|
||||
# kernel-grammar-parity test asserts node-kind-table equality at test time;
|
||||
# bump these together with the wasm side or that gate fails.
|
||||
tree-sitter-typescript = "0.23"
|
||||
tree-sitter-javascript = "0.25"
|
||||
|
||||
[build-dependencies]
|
||||
napi-build = "2"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
strip = "symbols"
|
||||
@@ -0,0 +1,3 @@
|
||||
fn main() {
|
||||
napi_build::setup();
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
; Seed query for the R1 scaffold (JavaScript / JSX grammar) — smoke-level
|
||||
; coverage only; R2 replaces this with the full port. See typescript.scm for
|
||||
; the capture convention.
|
||||
|
||||
(class_declaration name: (identifier) @name) @def.class
|
||||
(function_declaration name: (identifier) @name) @def.function
|
||||
(generator_function_declaration name: (identifier) @name) @def.function
|
||||
(method_definition name: (property_identifier) @name) @def.method
|
||||
|
||||
(call_expression function: (identifier) @ref.calls)
|
||||
(call_expression function: (member_expression property: (property_identifier) @ref.calls))
|
||||
(new_expression constructor: (identifier) @ref.instantiates)
|
||||
@@ -0,0 +1,21 @@
|
||||
; Seed query for the R1 scaffold — smoke-level coverage that proves the
|
||||
; buffer contract and emitter mechanics end to end. NOT extraction parity:
|
||||
; R2 replaces this with the full TypeScript/TSX port.
|
||||
;
|
||||
; Capture convention (see emitter.rs):
|
||||
; @def.<NodeKind> — the declaration node; pairs with @name in the pattern
|
||||
; @name — the declaration's name node
|
||||
; @ref.<EdgeKind> — a reference; the capture's own text is the name
|
||||
|
||||
(class_declaration name: (type_identifier) @name) @def.class
|
||||
(abstract_class_declaration name: (type_identifier) @name) @def.class
|
||||
(interface_declaration name: (type_identifier) @name) @def.interface
|
||||
(enum_declaration name: (identifier) @name) @def.enum
|
||||
(type_alias_declaration name: (type_identifier) @name) @def.type_alias
|
||||
(function_declaration name: (identifier) @name) @def.function
|
||||
(generator_function_declaration name: (identifier) @name) @def.function
|
||||
(method_definition name: (property_identifier) @name) @def.method
|
||||
|
||||
(call_expression function: (identifier) @ref.calls)
|
||||
(call_expression function: (member_expression property: (property_identifier) @ref.calls))
|
||||
(new_expression constructor: (identifier) @ref.instantiates)
|
||||
@@ -0,0 +1,387 @@
|
||||
//! Flat buffer contract — the ONE boundary crossing per file.
|
||||
//!
|
||||
//! The kernel returns five Buffers: meta, nodes, edges, refs, arena. All rows
|
||||
//! are fixed-width little-endian; every string is an (offset, len) pair into
|
||||
//! the UTF-8 arena. `OFFSET == NONE (0xFFFF_FFFF)` means "field absent".
|
||||
//!
|
||||
//! THIS FILE AND `src/extraction/kernel/layout.ts` MUST MATCH BYTE FOR BYTE.
|
||||
//! Any layout change bumps `KERNEL_ABI_VERSION` — the TS loader refuses a
|
||||
//! version it doesn't know and falls back to the wasm path.
|
||||
//!
|
||||
//! Layout (v1):
|
||||
//!
|
||||
//! meta (36 bytes):
|
||||
//! 0 u8 KERNEL_ABI_VERSION
|
||||
//! 1 [3] pad
|
||||
//! 4 u32 node count
|
||||
//! 8 u32 edge count
|
||||
//! 12 u32 ref count
|
||||
//! 16 u32 arena byte length
|
||||
//! 20 u32 errors-JSON arena offset (NONE = no errors)
|
||||
//! 24 u32 errors-JSON byte length
|
||||
//! 28 f64 kernel-side wall duration (ms) — introspection only; the TS
|
||||
//! wrapper measures the ExtractionResult.durationMs it reports
|
||||
//!
|
||||
//! node row (96 bytes):
|
||||
//! 0 u8 NodeKind index (NODE_KINDS order)
|
||||
//! 1 u8 visibility (0 absent, 1 public, 2 private, 3 protected, 4 internal)
|
||||
//! 2 u16 bool flags — bit pairs (present, value):
|
||||
//! 0/1 isExported, 2/3 isAsync, 4/5 isStatic, 6/7 isAbstract
|
||||
//! 4 u32 startLine (1-based)
|
||||
//! 8 u32 endLine
|
||||
//! 12 u32 startColumn (0-based)
|
||||
//! 16 u32 endColumn
|
||||
//! 20 str name
|
||||
//! 28 str qualifiedName
|
||||
//! 36 str id (kernel-computed: "kind:hash32", or "file:<path>" for the file node)
|
||||
//! 44 str docstring
|
||||
//! 52 str signature
|
||||
//! 60 str decorators (NUL-joined list)
|
||||
//! 68 str typeParameters (NUL-joined list)
|
||||
//! 76 str returnType
|
||||
//! 84 str extraJson (escape hatch: JSON of any extra Node props)
|
||||
//! 92 u32 metrics slot (reserved for Arc 3.2 per-node code metrics; 0)
|
||||
//!
|
||||
//! edge row (44 bytes):
|
||||
//! 0 u32 source node row index (NONE → use sourceIdStr)
|
||||
//! 4 u32 target node row index (NONE → use targetIdStr)
|
||||
//! 8 u8 EdgeKind index (EDGE_KINDS order)
|
||||
//! 9 u8 provenance (0 absent, 1 tree-sitter, 2 scip, 3 heuristic)
|
||||
//! 10 u16 pad
|
||||
//! 12 u32 line (NONE absent)
|
||||
//! 16 u32 column (NONE absent)
|
||||
//! 20 str metadataJson
|
||||
//! 28 str sourceIdStr
|
||||
//! 36 str targetIdStr
|
||||
//!
|
||||
//! ref row (40 bytes):
|
||||
//! 0 u32 fromNode row index (NONE → use fromNodeIdStr)
|
||||
//! 4 u8 ReferenceKind (EDGE_KINDS index, or 200 = function_ref)
|
||||
//! 5 [3] pad
|
||||
//! 8 u32 line (1-based)
|
||||
//! 12 u32 column (0-based)
|
||||
//! 16 str referenceName
|
||||
//! 24 str candidates (NUL-joined list)
|
||||
//! 32 str fromNodeIdStr
|
||||
|
||||
pub const KERNEL_ABI_VERSION: u8 = 1;
|
||||
pub const NONE: u32 = 0xFFFF_FFFF;
|
||||
|
||||
pub const META_SIZE: usize = 36;
|
||||
pub const NODE_ROW_SIZE: usize = 96;
|
||||
pub const EDGE_ROW_SIZE: usize = 44;
|
||||
pub const REF_ROW_SIZE: usize = 40;
|
||||
|
||||
/// Mirror of NODE_KINDS in src/types.ts — order is the wire contract.
|
||||
pub const NODE_KINDS: [&str; 22] = [
|
||||
"file",
|
||||
"module",
|
||||
"class",
|
||||
"struct",
|
||||
"interface",
|
||||
"trait",
|
||||
"protocol",
|
||||
"function",
|
||||
"method",
|
||||
"property",
|
||||
"field",
|
||||
"variable",
|
||||
"constant",
|
||||
"enum",
|
||||
"enum_member",
|
||||
"type_alias",
|
||||
"namespace",
|
||||
"parameter",
|
||||
"import",
|
||||
"export",
|
||||
"route",
|
||||
"component",
|
||||
];
|
||||
|
||||
/// Mirror of EDGE_KINDS in src/types.ts — order is the wire contract.
|
||||
pub const EDGE_KINDS: [&str; 12] = [
|
||||
"contains",
|
||||
"calls",
|
||||
"imports",
|
||||
"exports",
|
||||
"extends",
|
||||
"implements",
|
||||
"references",
|
||||
"type_of",
|
||||
"returns",
|
||||
"instantiates",
|
||||
"overrides",
|
||||
"decorates",
|
||||
];
|
||||
|
||||
/// ReferenceKind code for the internal-only `function_ref` (#756).
|
||||
pub const FUNCTION_REF_CODE: u8 = 200;
|
||||
|
||||
pub fn node_kind_index(kind: &str) -> Option<u8> {
|
||||
NODE_KINDS.iter().position(|k| *k == kind).map(|i| i as u8)
|
||||
}
|
||||
|
||||
pub fn edge_kind_index(kind: &str) -> Option<u8> {
|
||||
EDGE_KINDS.iter().position(|k| *k == kind).map(|i| i as u8)
|
||||
}
|
||||
|
||||
/// (offset, len) arena reference. `NONE_STR` encodes an absent field.
|
||||
pub type StrRef = (u32, u32);
|
||||
pub const NONE_STR: StrRef = (NONE, 0);
|
||||
|
||||
/// UTF-8 string arena. Strings are appended verbatim; no dedup (per-file
|
||||
/// buffers are transient and small — intern later if profiling says so).
|
||||
#[derive(Default)]
|
||||
pub struct Arena {
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Arena {
|
||||
pub fn put(&mut self, s: &str) -> StrRef {
|
||||
let off = self.buf.len() as u32;
|
||||
self.buf.extend_from_slice(s.as_bytes());
|
||||
(off, s.len() as u32)
|
||||
}
|
||||
|
||||
/// Not used by the seed emitter yet — R2 (docstring/signature/etc.). Kept
|
||||
/// so the arena API is complete alongside the layout it feeds.
|
||||
#[allow(dead_code)]
|
||||
pub fn put_opt(&mut self, s: Option<&str>) -> StrRef {
|
||||
match s {
|
||||
Some(s) => self.put(s),
|
||||
None => NONE_STR,
|
||||
}
|
||||
}
|
||||
|
||||
/// NUL-joined list; absent when the list is empty. (R2 surface: decorators,
|
||||
/// typeParameters, candidates.)
|
||||
#[allow(dead_code)]
|
||||
pub fn put_list(&mut self, items: &[String]) -> StrRef {
|
||||
if items.is_empty() {
|
||||
return NONE_STR;
|
||||
}
|
||||
let joined = items.join("\0");
|
||||
self.put(&joined)
|
||||
}
|
||||
|
||||
pub fn len(&self) -> u32 {
|
||||
self.buf.len() as u32
|
||||
}
|
||||
|
||||
pub fn into_vec(self) -> Vec<u8> {
|
||||
self.buf
|
||||
}
|
||||
}
|
||||
|
||||
/// Tri-state booleans packed as (present, value) bit pairs.
|
||||
#[derive(Default, Clone, Copy)]
|
||||
pub struct BoolFlags(pub u16);
|
||||
|
||||
impl BoolFlags {
|
||||
pub fn set(&mut self, pair: u16, value: bool) {
|
||||
self.0 |= 1 << (pair * 2);
|
||||
if value {
|
||||
self.0 |= 1 << (pair * 2 + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub const FLAG_IS_EXPORTED: u16 = 0;
|
||||
#[allow(dead_code)] // R2 surface — part of the v1 wire contract
|
||||
pub const FLAG_IS_ASYNC: u16 = 1;
|
||||
#[allow(dead_code)] // R2 surface — part of the v1 wire contract
|
||||
pub const FLAG_IS_STATIC: u16 = 2;
|
||||
#[allow(dead_code)] // R2 surface — part of the v1 wire contract
|
||||
pub const FLAG_IS_ABSTRACT: u16 = 3;
|
||||
|
||||
pub struct NodeRow {
|
||||
pub kind: u8,
|
||||
pub visibility: u8,
|
||||
pub flags: BoolFlags,
|
||||
pub start_line: u32,
|
||||
pub end_line: u32,
|
||||
pub start_column: u32,
|
||||
pub end_column: u32,
|
||||
pub name: StrRef,
|
||||
pub qualified_name: StrRef,
|
||||
pub id: StrRef,
|
||||
pub docstring: StrRef,
|
||||
pub signature: StrRef,
|
||||
pub decorators: StrRef,
|
||||
pub type_parameters: StrRef,
|
||||
pub return_type: StrRef,
|
||||
pub extra_json: StrRef,
|
||||
}
|
||||
|
||||
pub struct EdgeRow {
|
||||
pub source_idx: u32,
|
||||
pub target_idx: u32,
|
||||
pub kind: u8,
|
||||
pub provenance: u8,
|
||||
pub line: u32,
|
||||
pub column: u32,
|
||||
pub metadata_json: StrRef,
|
||||
pub source_id_str: StrRef,
|
||||
pub target_id_str: StrRef,
|
||||
}
|
||||
|
||||
pub struct RefRow {
|
||||
pub from_idx: u32,
|
||||
pub kind: u8,
|
||||
pub line: u32,
|
||||
pub column: u32,
|
||||
pub reference_name: StrRef,
|
||||
pub candidates: StrRef,
|
||||
pub from_id_str: StrRef,
|
||||
}
|
||||
|
||||
fn push_str_ref(buf: &mut Vec<u8>, r: StrRef) {
|
||||
buf.extend_from_slice(&r.0.to_le_bytes());
|
||||
buf.extend_from_slice(&r.1.to_le_bytes());
|
||||
}
|
||||
|
||||
pub struct Tables {
|
||||
pub nodes: Vec<u8>,
|
||||
pub edges: Vec<u8>,
|
||||
pub refs: Vec<u8>,
|
||||
pub node_count: u32,
|
||||
pub edge_count: u32,
|
||||
pub ref_count: u32,
|
||||
}
|
||||
|
||||
impl Default for Tables {
|
||||
fn default() -> Self {
|
||||
Tables {
|
||||
nodes: Vec::with_capacity(NODE_ROW_SIZE * 64),
|
||||
edges: Vec::with_capacity(EDGE_ROW_SIZE * 64),
|
||||
refs: Vec::with_capacity(REF_ROW_SIZE * 64),
|
||||
node_count: 0,
|
||||
edge_count: 0,
|
||||
ref_count: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Tables {
|
||||
pub fn push_node(&mut self, r: &NodeRow) -> u32 {
|
||||
let buf = &mut self.nodes;
|
||||
buf.push(r.kind);
|
||||
buf.push(r.visibility);
|
||||
buf.extend_from_slice(&r.flags.0.to_le_bytes());
|
||||
buf.extend_from_slice(&r.start_line.to_le_bytes());
|
||||
buf.extend_from_slice(&r.end_line.to_le_bytes());
|
||||
buf.extend_from_slice(&r.start_column.to_le_bytes());
|
||||
buf.extend_from_slice(&r.end_column.to_le_bytes());
|
||||
push_str_ref(buf, r.name);
|
||||
push_str_ref(buf, r.qualified_name);
|
||||
push_str_ref(buf, r.id);
|
||||
push_str_ref(buf, r.docstring);
|
||||
push_str_ref(buf, r.signature);
|
||||
push_str_ref(buf, r.decorators);
|
||||
push_str_ref(buf, r.type_parameters);
|
||||
push_str_ref(buf, r.return_type);
|
||||
push_str_ref(buf, r.extra_json);
|
||||
buf.extend_from_slice(&0u32.to_le_bytes()); // metrics slot (Arc 3.2)
|
||||
let idx = self.node_count;
|
||||
self.node_count += 1;
|
||||
idx
|
||||
}
|
||||
|
||||
pub fn push_edge(&mut self, r: &EdgeRow) {
|
||||
let buf = &mut self.edges;
|
||||
buf.extend_from_slice(&r.source_idx.to_le_bytes());
|
||||
buf.extend_from_slice(&r.target_idx.to_le_bytes());
|
||||
buf.push(r.kind);
|
||||
buf.push(r.provenance);
|
||||
buf.extend_from_slice(&0u16.to_le_bytes()); // pad
|
||||
buf.extend_from_slice(&r.line.to_le_bytes());
|
||||
buf.extend_from_slice(&r.column.to_le_bytes());
|
||||
push_str_ref(buf, r.metadata_json);
|
||||
push_str_ref(buf, r.source_id_str);
|
||||
push_str_ref(buf, r.target_id_str);
|
||||
self.edge_count += 1;
|
||||
}
|
||||
|
||||
pub fn push_ref(&mut self, r: &RefRow) {
|
||||
let buf = &mut self.refs;
|
||||
buf.extend_from_slice(&r.from_idx.to_le_bytes());
|
||||
buf.push(r.kind);
|
||||
buf.extend_from_slice(&[0u8; 3]); // pad
|
||||
buf.extend_from_slice(&r.line.to_le_bytes());
|
||||
buf.extend_from_slice(&r.column.to_le_bytes());
|
||||
push_str_ref(buf, r.reference_name);
|
||||
push_str_ref(buf, r.candidates);
|
||||
push_str_ref(buf, r.from_id_str);
|
||||
self.ref_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_meta(t: &Tables, arena_len: u32, errors_json: StrRef, duration_ms: f64) -> Vec<u8> {
|
||||
let mut m = Vec::with_capacity(META_SIZE);
|
||||
m.push(KERNEL_ABI_VERSION);
|
||||
m.extend_from_slice(&[0u8; 3]);
|
||||
m.extend_from_slice(&t.node_count.to_le_bytes());
|
||||
m.extend_from_slice(&t.edge_count.to_le_bytes());
|
||||
m.extend_from_slice(&t.ref_count.to_le_bytes());
|
||||
m.extend_from_slice(&arena_len.to_le_bytes());
|
||||
m.extend_from_slice(&errors_json.0.to_le_bytes());
|
||||
m.extend_from_slice(&errors_json.1.to_le_bytes());
|
||||
m.extend_from_slice(&duration_ms.to_le_bytes());
|
||||
debug_assert_eq!(m.len(), META_SIZE);
|
||||
m
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn row_sizes_match_constants() {
|
||||
let mut t = Tables::default();
|
||||
let mut a = Arena::default();
|
||||
let name = a.put("x");
|
||||
t.push_node(&NodeRow {
|
||||
kind: 0,
|
||||
visibility: 0,
|
||||
flags: BoolFlags::default(),
|
||||
start_line: 1,
|
||||
end_line: 1,
|
||||
start_column: 0,
|
||||
end_column: 0,
|
||||
name,
|
||||
qualified_name: name,
|
||||
id: name,
|
||||
docstring: NONE_STR,
|
||||
signature: NONE_STR,
|
||||
decorators: NONE_STR,
|
||||
type_parameters: NONE_STR,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
assert_eq!(t.nodes.len(), NODE_ROW_SIZE);
|
||||
t.push_edge(&EdgeRow {
|
||||
source_idx: 0,
|
||||
target_idx: 0,
|
||||
kind: 0,
|
||||
provenance: 0,
|
||||
line: NONE,
|
||||
column: NONE,
|
||||
metadata_json: NONE_STR,
|
||||
source_id_str: NONE_STR,
|
||||
target_id_str: NONE_STR,
|
||||
});
|
||||
assert_eq!(t.edges.len(), EDGE_ROW_SIZE);
|
||||
t.push_ref(&RefRow {
|
||||
from_idx: 0,
|
||||
kind: 1,
|
||||
line: 1,
|
||||
column: 0,
|
||||
reference_name: name,
|
||||
candidates: NONE_STR,
|
||||
from_id_str: NONE_STR,
|
||||
});
|
||||
assert_eq!(t.refs.len(), REF_ROW_SIZE);
|
||||
let meta = build_meta(&t, a.len(), NONE_STR, 0.0);
|
||||
assert_eq!(meta.len(), META_SIZE);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
//! Generic query-driven emitter: parse the file, run the language's `.scm`
|
||||
//! query, and emit flat rows. The whole tree walk happens native-side; the
|
||||
//! only JS boundary crossing is the returned buffers.
|
||||
//!
|
||||
//! Mechanics mirrored from `TreeSitterExtractor` (src/extraction/tree-sitter.ts):
|
||||
//! - node row 0 is the file node (`file:<path>`, endLine = newline count + 1,
|
||||
//! isExported present+false — byte-parity with the TS file node);
|
||||
//! - definitions form a scope stack by byte-range nesting; qualifiedName is
|
||||
//! the stack's names joined with `::` (file excluded);
|
||||
//! - every definition gets a `contains` edge from its parent scope (the
|
||||
//! file node when top-level);
|
||||
//! - references attach to the innermost enclosing definition, falling back
|
||||
//! to the file node — same as the TS extractor's nodeStack semantics;
|
||||
//! - definitions with empty names are skipped (issue #42 semantics).
|
||||
|
||||
use crate::buffers::{
|
||||
build_meta, edge_kind_index, node_kind_index, Arena, BoolFlags, EdgeRow, NodeRow, RefRow,
|
||||
Tables, FLAG_IS_EXPORTED, FUNCTION_REF_CODE, NODE_KINDS, NONE, NONE_STR,
|
||||
};
|
||||
use crate::ids;
|
||||
use crate::langs::LangSpec;
|
||||
use streaming_iterator::StreamingIterator;
|
||||
use tree_sitter::{Node, Parser, QueryCursor};
|
||||
|
||||
pub struct EmitOut {
|
||||
pub meta: Vec<u8>,
|
||||
pub nodes: Vec<u8>,
|
||||
pub edges: Vec<u8>,
|
||||
pub refs: Vec<u8>,
|
||||
pub arena: Vec<u8>,
|
||||
}
|
||||
|
||||
/// What a query capture name means. Resolved once per query.
|
||||
#[derive(Clone, Copy)]
|
||||
enum Role {
|
||||
/// `@def.<NodeKind>` — value is the NODE_KINDS index.
|
||||
Def(u8),
|
||||
/// `@name` — the paired definition's name node.
|
||||
Name,
|
||||
/// `@ref.<EdgeKind>` / `@ref.function_ref` — value is the wire code.
|
||||
Ref(u8),
|
||||
/// Helper captures (`@_anchor` etc.) — ignored.
|
||||
Ignore,
|
||||
}
|
||||
|
||||
fn resolve_roles(capture_names: &[&str], lang: &str) -> Result<Vec<Role>, String> {
|
||||
capture_names
|
||||
.iter()
|
||||
.map(|name| {
|
||||
if let Some(kind) = name.strip_prefix("def.") {
|
||||
let idx = node_kind_index(kind)
|
||||
.ok_or_else(|| format!("{lang}: unknown NodeKind in capture @{name}"))?;
|
||||
Ok(Role::Def(idx))
|
||||
} else if let Some(kind) = name.strip_prefix("ref.") {
|
||||
if kind == "function_ref" {
|
||||
return Ok(Role::Ref(FUNCTION_REF_CODE));
|
||||
}
|
||||
let idx = edge_kind_index(kind)
|
||||
.ok_or_else(|| format!("{lang}: unknown EdgeKind in capture @{name}"))?;
|
||||
Ok(Role::Ref(idx))
|
||||
} else if *name == "name" {
|
||||
Ok(Role::Name)
|
||||
} else {
|
||||
Ok(Role::Ignore)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
struct Def {
|
||||
kind: u8,
|
||||
name_start: usize,
|
||||
name_end: usize,
|
||||
start_byte: usize,
|
||||
end_byte: usize,
|
||||
start_line: u32,
|
||||
end_line: u32,
|
||||
start_column: u32,
|
||||
end_column: u32,
|
||||
/// Node-table row index, assigned during the scope sweep.
|
||||
row: u32,
|
||||
}
|
||||
|
||||
struct RefCap {
|
||||
kind: u8,
|
||||
name_start: usize,
|
||||
name_end: usize,
|
||||
start_byte: usize,
|
||||
line: u32,
|
||||
column: u32,
|
||||
}
|
||||
|
||||
pub fn extract(file_path: &str, source: &str, spec: &LangSpec) -> Result<EmitOut, String> {
|
||||
let t0 = std::time::Instant::now();
|
||||
|
||||
let mut parser = Parser::new();
|
||||
parser
|
||||
.set_language(spec.language())
|
||||
.map_err(|e| format!("set_language({}) failed: {e}", spec.name))?;
|
||||
let tree = parser
|
||||
.parse(source, None)
|
||||
.ok_or_else(|| "parser returned null tree".to_string())?;
|
||||
let root = tree.root_node();
|
||||
|
||||
let query = spec.query()?;
|
||||
let roles = resolve_roles(&query.capture_names(), spec.name)?;
|
||||
|
||||
// ---- Collect definition + reference captures from the query. ----
|
||||
let mut defs: Vec<Def> = Vec::new();
|
||||
let mut refs: Vec<RefCap> = Vec::new();
|
||||
// A node can match several patterns (e.g. nested alternations); first
|
||||
// pattern wins, mirroring the TS walk's one-node-one-symbol behaviour.
|
||||
let mut seen_defs = std::collections::HashSet::<usize>::new();
|
||||
|
||||
let mut cursor = QueryCursor::new();
|
||||
let mut matches = cursor.matches(query, root, source.as_bytes());
|
||||
while let Some(m) = matches.next() {
|
||||
let mut def_node: Option<(Node, u8)> = None;
|
||||
let mut name_node: Option<Node> = None;
|
||||
for cap in m.captures {
|
||||
match roles[cap.index as usize] {
|
||||
Role::Def(kind) => def_node = Some((cap.node, kind)),
|
||||
Role::Name => name_node = Some(cap.node),
|
||||
Role::Ref(kind) => {
|
||||
let p = cap.node.start_position();
|
||||
refs.push(RefCap {
|
||||
kind,
|
||||
name_start: cap.node.start_byte(),
|
||||
name_end: cap.node.end_byte(),
|
||||
start_byte: cap.node.start_byte(),
|
||||
line: p.row as u32 + 1,
|
||||
column: p.column as u32,
|
||||
});
|
||||
}
|
||||
Role::Ignore => {}
|
||||
}
|
||||
}
|
||||
if let (Some((node, kind)), Some(name)) = (def_node, name_node) {
|
||||
// Empty names are not meaningful symbols (issue #42).
|
||||
if name.end_byte() > name.start_byte() && seen_defs.insert(node.id()) {
|
||||
let sp = node.start_position();
|
||||
let ep = node.end_position();
|
||||
defs.push(Def {
|
||||
kind,
|
||||
name_start: name.start_byte(),
|
||||
name_end: name.end_byte(),
|
||||
start_byte: node.start_byte(),
|
||||
end_byte: node.end_byte(),
|
||||
start_line: sp.row as u32 + 1,
|
||||
end_line: ep.row as u32 + 1,
|
||||
start_column: sp.column as u32,
|
||||
end_column: ep.column as u32,
|
||||
row: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deterministic pre-order regardless of query-match ordering.
|
||||
defs.sort_by(|a, b| {
|
||||
a.start_byte
|
||||
.cmp(&b.start_byte)
|
||||
.then(b.end_byte.cmp(&a.end_byte))
|
||||
});
|
||||
refs.sort_by_key(|r| r.start_byte);
|
||||
|
||||
// ---- Emit rows: file node first, then the scope-stack sweep. ----
|
||||
let mut arena = Arena::default();
|
||||
let mut tables = Tables::default();
|
||||
|
||||
let line_count = source.bytes().filter(|b| *b == b'\n').count() as u32 + 1;
|
||||
let base_name = file_path.rsplit(['/', '\\']).next().unwrap_or(file_path);
|
||||
let mut file_flags = BoolFlags::default();
|
||||
file_flags.set(FLAG_IS_EXPORTED, false);
|
||||
let file_id = arena.put(&ids::file_node_id(file_path));
|
||||
let file_name = arena.put(base_name);
|
||||
let file_qn = arena.put(file_path);
|
||||
tables.push_node(&NodeRow {
|
||||
kind: node_kind_index("file").unwrap(),
|
||||
visibility: 0,
|
||||
flags: file_flags,
|
||||
start_line: 1,
|
||||
end_line: line_count,
|
||||
start_column: 0,
|
||||
end_column: 0,
|
||||
name: file_name,
|
||||
qualified_name: file_qn,
|
||||
id: file_id,
|
||||
docstring: NONE_STR,
|
||||
signature: NONE_STR,
|
||||
decorators: NONE_STR,
|
||||
type_parameters: NONE_STR,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
|
||||
// Merged sweep over definitions and references in byte order, maintaining
|
||||
// the scope stack (indices into `defs`).
|
||||
let mut stack: Vec<usize> = Vec::new();
|
||||
let mut ref_i = 0usize;
|
||||
|
||||
fn pop_to(stack: &mut Vec<usize>, defs: &[Def], byte: usize) {
|
||||
while let Some(&top) = stack.last() {
|
||||
if defs[top].end_byte <= byte {
|
||||
stack.pop();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let emit_ref = |r: &RefCap, stack: &[usize], defs: &[Def], arena: &mut Arena, tables: &mut Tables| {
|
||||
let from_idx = stack.last().map(|&i| defs[i].row).unwrap_or(0);
|
||||
let name = arena.put(&source[r.name_start..r.name_end]);
|
||||
tables.push_ref(&RefRow {
|
||||
from_idx,
|
||||
kind: r.kind,
|
||||
line: r.line,
|
||||
column: r.column,
|
||||
reference_name: name,
|
||||
candidates: NONE_STR,
|
||||
from_id_str: NONE_STR,
|
||||
});
|
||||
};
|
||||
|
||||
for i in 0..defs.len() {
|
||||
let def_start = defs[i].start_byte;
|
||||
while ref_i < refs.len() && refs[ref_i].start_byte < def_start {
|
||||
pop_to(&mut stack, &defs, refs[ref_i].start_byte);
|
||||
emit_ref(&refs[ref_i], &stack, &defs, &mut arena, &mut tables);
|
||||
ref_i += 1;
|
||||
}
|
||||
pop_to(&mut stack, &defs, def_start);
|
||||
|
||||
let name = &source[defs[i].name_start..defs[i].name_end];
|
||||
let kind_str = NODE_KINDS[defs[i].kind as usize];
|
||||
// qualifiedName = enclosing definition names + own name, `::`-joined
|
||||
// (buildQualifiedName semantics; file node excluded).
|
||||
let mut qn = String::new();
|
||||
for &s in stack.iter() {
|
||||
qn.push_str(&source[defs[s].name_start..defs[s].name_end]);
|
||||
qn.push_str("::");
|
||||
}
|
||||
qn.push_str(name);
|
||||
|
||||
let id = ids::node_id(file_path, kind_str, name, defs[i].start_line);
|
||||
let id_ref = arena.put(&id);
|
||||
let name_ref = arena.put(name);
|
||||
let qn_ref = arena.put(&qn);
|
||||
let row = tables.push_node(&NodeRow {
|
||||
kind: defs[i].kind,
|
||||
visibility: 0,
|
||||
flags: BoolFlags::default(),
|
||||
start_line: defs[i].start_line,
|
||||
end_line: defs[i].end_line,
|
||||
start_column: defs[i].start_column,
|
||||
end_column: defs[i].end_column,
|
||||
name: name_ref,
|
||||
qualified_name: qn_ref,
|
||||
id: id_ref,
|
||||
docstring: NONE_STR,
|
||||
signature: NONE_STR,
|
||||
decorators: NONE_STR,
|
||||
type_parameters: NONE_STR,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
defs[i].row = row;
|
||||
|
||||
let parent_row = stack.last().map(|&s| defs[s].row).unwrap_or(0);
|
||||
tables.push_edge(&EdgeRow {
|
||||
source_idx: parent_row,
|
||||
target_idx: row,
|
||||
kind: edge_kind_index("contains").unwrap(),
|
||||
provenance: 0,
|
||||
line: NONE,
|
||||
column: NONE,
|
||||
metadata_json: NONE_STR,
|
||||
source_id_str: NONE_STR,
|
||||
target_id_str: NONE_STR,
|
||||
});
|
||||
|
||||
stack.push(i);
|
||||
}
|
||||
while ref_i < refs.len() {
|
||||
pop_to(&mut stack, &defs, refs[ref_i].start_byte);
|
||||
emit_ref(&refs[ref_i], &stack, &defs, &mut arena, &mut tables);
|
||||
ref_i += 1;
|
||||
}
|
||||
|
||||
let duration_ms = t0.elapsed().as_secs_f64() * 1000.0;
|
||||
let meta = build_meta(&tables, arena.len(), NONE_STR, duration_ms);
|
||||
Ok(EmitOut {
|
||||
meta,
|
||||
nodes: tables.nodes,
|
||||
edges: tables.edges,
|
||||
refs: tables.refs,
|
||||
arena: arena.into_vec(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
//! Node-ID generation — MUST produce byte-identical output to
|
||||
//! `generateNodeId` in `src/extraction/tree-sitter-helpers.ts`:
|
||||
//!
|
||||
//! `${kind}:${sha256(`${filePath}:${kind}:${name}:${line}`).hex[0..32]}`
|
||||
//!
|
||||
//! and the file-node special case in `TreeSitterExtractor.extract()`:
|
||||
//!
|
||||
//! `file:${filePath}`
|
||||
//!
|
||||
//! Node identity is how the wasm path and the kernel path agree on the same
|
||||
//! graph — a drift here breaks every edge. Pinned by the node-id parity test
|
||||
//! in `__tests__/kernel-scaffold.test.ts`.
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub fn node_id(file_path: &str, kind: &str, name: &str, line: u32) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(file_path.as_bytes());
|
||||
hasher.update(b":");
|
||||
hasher.update(kind.as_bytes());
|
||||
hasher.update(b":");
|
||||
hasher.update(name.as_bytes());
|
||||
hasher.update(b":");
|
||||
hasher.update(line.to_string().as_bytes());
|
||||
let digest = hasher.finalize();
|
||||
// 32 hex chars = first 16 bytes.
|
||||
let mut hex = String::with_capacity(kind.len() + 1 + 32);
|
||||
hex.push_str(kind);
|
||||
hex.push(':');
|
||||
for b in &digest[..16] {
|
||||
hex.push_str(&format!("{b:02x}"));
|
||||
}
|
||||
hex
|
||||
}
|
||||
|
||||
pub fn file_node_id(file_path: &str) -> String {
|
||||
format!("file:{file_path}")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn matches_known_ts_output() {
|
||||
// Pinned vector: node -e "crypto.createHash('sha256')
|
||||
// .update('src/a.ts:function:foo:3').digest('hex').substring(0,32)"
|
||||
assert_eq!(
|
||||
node_id("src/a.ts", "function", "foo", 3),
|
||||
"function:bfb15544fed707794274a5c61006ea7b"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
//! Per-language specs: grammar + `.scm` query + (later) per-language config.
|
||||
//!
|
||||
//! Tier-1 languages are meant to be *mostly* a query file plus a small config
|
||||
//! here; logic queries can't express stays TS-side as a per-language `post()`
|
||||
//! hook over the returned buffers (see `src/extraction/kernel/route.ts`).
|
||||
//!
|
||||
//! Language strings are codegraph `Language` values (src/types.ts), not
|
||||
//! grammar names — `tsx` and `jsx` are separate entries that reuse another
|
||||
//! entry's grammar exactly like `WASM_GRAMMAR_FILES` does on the wasm path.
|
||||
|
||||
use std::sync::OnceLock;
|
||||
use tree_sitter::{Language, Query};
|
||||
|
||||
pub struct LangSpec {
|
||||
/// codegraph Language string (src/types.ts).
|
||||
pub name: &'static str,
|
||||
get_language: fn() -> Language,
|
||||
query_src: &'static str,
|
||||
language: OnceLock<Language>,
|
||||
query: OnceLock<Result<Query, String>>,
|
||||
}
|
||||
|
||||
impl LangSpec {
|
||||
const fn new(name: &'static str, get_language: fn() -> Language, query_src: &'static str) -> Self {
|
||||
LangSpec {
|
||||
name,
|
||||
get_language,
|
||||
query_src,
|
||||
language: OnceLock::new(),
|
||||
query: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn language(&self) -> &Language {
|
||||
self.language.get_or_init(self.get_language)
|
||||
}
|
||||
|
||||
pub fn query(&self) -> Result<&Query, String> {
|
||||
self.query
|
||||
.get_or_init(|| {
|
||||
Query::new(self.language(), self.query_src)
|
||||
.map_err(|e| format!("query compile failed for {}: {e}", self.name))
|
||||
})
|
||||
.as_ref()
|
||||
.map_err(|e| e.clone())
|
||||
}
|
||||
}
|
||||
|
||||
fn ts_language() -> Language {
|
||||
tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()
|
||||
}
|
||||
|
||||
fn tsx_language() -> Language {
|
||||
tree_sitter_typescript::LANGUAGE_TSX.into()
|
||||
}
|
||||
|
||||
fn js_language() -> Language {
|
||||
tree_sitter_javascript::LANGUAGE.into()
|
||||
}
|
||||
|
||||
static TYPESCRIPT: LangSpec = LangSpec::new(
|
||||
"typescript",
|
||||
ts_language,
|
||||
include_str!("../queries/typescript.scm"),
|
||||
);
|
||||
static TSX: LangSpec = LangSpec::new("tsx", tsx_language, include_str!("../queries/typescript.scm"));
|
||||
static JAVASCRIPT: LangSpec = LangSpec::new(
|
||||
"javascript",
|
||||
js_language,
|
||||
include_str!("../queries/javascript.scm"),
|
||||
);
|
||||
static JSX: LangSpec = LangSpec::new("jsx", js_language, include_str!("../queries/javascript.scm"));
|
||||
|
||||
pub static ALL: [&LangSpec; 4] = [&TYPESCRIPT, &TSX, &JAVASCRIPT, &JSX];
|
||||
|
||||
pub fn spec_for(language: &str) -> Option<&'static LangSpec> {
|
||||
ALL.iter().find(|s| s.name == language).copied()
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
//! codegraph-kernel — native extraction kernel (napi-rs).
|
||||
//!
|
||||
//! Replaces ONLY the parse+extract walk inside the parse workers, behind the
|
||||
//! existing `ExtractionResult` contract. Input `(filePath, content, language)`
|
||||
//! per file; output flat typed buffers — one boundary crossing per file.
|
||||
//! Everything downstream (resolution, synthesis, frameworks, MCP) is
|
||||
//! untouched and consumes the decoded result exactly as before.
|
||||
//!
|
||||
//! Calls are synchronous by design: the existing `ParseWorkerPool` workers
|
||||
//! already parallelize per-file, so each worker thread drives its own kernel
|
||||
//! call (do NOT rebuild the pool on the Rust side — see the migration plan §3).
|
||||
|
||||
#![deny(clippy::all)]
|
||||
|
||||
mod buffers;
|
||||
mod emitter;
|
||||
mod ids;
|
||||
mod langs;
|
||||
|
||||
use napi::bindgen_prelude::*;
|
||||
use napi_derive::napi;
|
||||
|
||||
/// The five flat tables for one file. See buffers.rs for the byte layout;
|
||||
/// `src/extraction/kernel/layout.ts` is the TS mirror.
|
||||
#[napi(object)]
|
||||
pub struct ExtractBuffers {
|
||||
pub meta: Buffer,
|
||||
pub nodes: Buffer,
|
||||
pub edges: Buffer,
|
||||
pub refs: Buffer,
|
||||
pub arena: Buffer,
|
||||
}
|
||||
|
||||
/// Wire-contract description — the TS loader verifies this against
|
||||
/// src/types.ts before routing anything to the kernel, so an out-of-date
|
||||
/// `.node` degrades to the wasm path instead of mis-decoding.
|
||||
#[napi(object)]
|
||||
pub struct ContractInfo {
|
||||
pub abi_version: u32,
|
||||
pub kernel_version: String,
|
||||
pub node_kinds: Vec<String>,
|
||||
pub edge_kinds: Vec<String>,
|
||||
/// Languages this binary can extract (routing is still TS-side policy).
|
||||
pub languages: Vec<String>,
|
||||
}
|
||||
|
||||
/// Grammar identity for the grammar-source-parity gate: the wasm grammar and
|
||||
/// the native grammar must expose identical node-kind/field tables, or
|
||||
/// kernel-vs-fallback routing would be non-deterministic.
|
||||
#[napi(object)]
|
||||
pub struct GrammarInfo {
|
||||
pub abi_version: u32,
|
||||
pub node_kind_count: u32,
|
||||
pub field_count: u32,
|
||||
pub node_kinds: Vec<String>,
|
||||
pub field_names: Vec<String>,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn contract_info() -> ContractInfo {
|
||||
ContractInfo {
|
||||
abi_version: buffers::KERNEL_ABI_VERSION as u32,
|
||||
kernel_version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
node_kinds: buffers::NODE_KINDS.iter().map(|s| s.to_string()).collect(),
|
||||
edge_kinds: buffers::EDGE_KINDS.iter().map(|s| s.to_string()).collect(),
|
||||
languages: langs::ALL.iter().map(|s| s.name.to_string()).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn grammar_info(language: String) -> Option<GrammarInfo> {
|
||||
let spec = langs::spec_for(&language)?;
|
||||
let lang = spec.language();
|
||||
let node_kind_count = lang.node_kind_count();
|
||||
let field_count = lang.field_count();
|
||||
let node_kinds = (0..node_kind_count)
|
||||
.map(|i| lang.node_kind_for_id(i as u16).unwrap_or("").to_string())
|
||||
.collect();
|
||||
// Field ids are 1-based in tree-sitter.
|
||||
let field_names = (1..=field_count)
|
||||
.map(|i| lang.field_name_for_id(i as u16).unwrap_or("").to_string())
|
||||
.collect();
|
||||
Some(GrammarInfo {
|
||||
abi_version: lang.abi_version() as u32,
|
||||
node_kind_count: node_kind_count as u32,
|
||||
field_count: field_count as u32,
|
||||
node_kinds,
|
||||
field_names,
|
||||
})
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn extract_file(file_path: String, content: String, language: String) -> Result<ExtractBuffers> {
|
||||
let spec = langs::spec_for(&language)
|
||||
.ok_or_else(|| Error::from_reason(format!("kernel does not support language: {language}")))?;
|
||||
let out = emitter::extract(&file_path, &content, spec).map_err(Error::from_reason)?;
|
||||
Ok(ExtractBuffers {
|
||||
meta: out.meta.into(),
|
||||
nodes: out.nodes.into(),
|
||||
edges: out.edges.into(),
|
||||
refs: out.refs.into(),
|
||||
arena: out.arena.into(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
# Rust extraction-kernel migration plan + post-kernel roadmap
|
||||
|
||||
**Audience:** the agent/engineer executing the native-kernel project. Self-contained handoff:
|
||||
context, current state, per-language tracker, gates, and the follow-on roadmap.
|
||||
**Companion:** `docs/design/native-extraction-kernel.md` (architecture + spike detail).
|
||||
**Written:** 2026-07-16, after the perf arc that shipped #1305, #1320, #1321, #1322, #1323.
|
||||
|
||||
---
|
||||
|
||||
## 0. Execution order (the whole plan as one checklist)
|
||||
|
||||
Work top to bottom; each step has a section below with the detail.
|
||||
|
||||
- [x] **R1. Scaffold the napi-rs crate** (`codegraph-kernel`): buffer contract, generic
|
||||
`.scm` emitter, build-pipeline integration, `CODEGRAPH_KERNEL=0` kill switch,
|
||||
wasm fallback, grammar-source-parity CI. (§3) — **done 2026-07-16, see §3a.**
|
||||
- [ ] **R2. Port TypeScript/JavaScript extraction** (tsx/jsx included) as language one. (§4)
|
||||
- [ ] **R3. Run TS/JS through the equivalence gate** — graph parity, retrieval
|
||||
invariants, agent A/B, perf + control repo. Ship behind the env flag, then default-on. (§5)
|
||||
- [ ] **R4. Port Java** → re-run the dubbo benchmark → the cbm-parity headline. (§4, §6)
|
||||
- [ ] **R5. Port Python, Go.** (§4)
|
||||
- [ ] **R6. Kernel-scale re-validation** in the cg1212 container (expect parse 6m → ~2m). (§6)
|
||||
- [ ] **R7. Long-tail languages opportunistically** per the tracker; T3 may stay TS forever. (§4)
|
||||
- [ ] **P1. Kernel-scale resolution speed** — the 19.5-min sequential wall at 2M nodes. (§7a)
|
||||
- [ ] **P2. Arc 3, graph richness** — in priority order: test edges → code metrics →
|
||||
read/write refs → raises → doc sections → IaC nodes. Each behind the standard gate. (§7b)
|
||||
- [ ] **P3. Parked items** — only with explicit maintainer approval. (§7c)
|
||||
|
||||
---
|
||||
|
||||
## 1. Mission and the numbers that motivate it
|
||||
|
||||
CodeGraph's remaining fresh-index gap vs codebase-memory-mcp (cbm) is the parse+extract
|
||||
phase, and its floor is per-node JS↔WASM marshaling — proven, not suspected:
|
||||
|
||||
| Measurement (2026-07-16, M3 Pro) | Result |
|
||||
|---|---|
|
||||
| dubbo (4,402 Java files) parse-loop, current 7-wasm-worker pipeline | 4,700ms |
|
||||
| Same files, Rust tree-sitter parse+walk, rayon (spike) | **202ms** |
|
||||
| Same, single Rust thread | 1,067ms |
|
||||
| dubbo fresh init today / cbm | 11.1s / 7.1s (1.55×) |
|
||||
| Linux kernel, same 2-CPU/6GB container | **we complete 27min; cbm dies at 0.16%, twice** |
|
||||
|
||||
Spike source: session scratchpad `cg-kernel-spike/` (tree-sitter 0.25 + tree-sitter-java,
|
||||
TreeCursor walk touching kind/range/name-field, flat-row output). Reproduce before starting —
|
||||
it's ~80 lines and doubles as the emitter's seed.
|
||||
|
||||
Expected end state: parse-loop 4.7s → ~1.0–1.5s on dubbo-class repos → total ≈ 7.5s,
|
||||
**parity with cbm on their best surface**, while keeping every win we already hold
|
||||
(sync 2.4–2.8×, agent A/B decisive, call-graph density 1.3–2.3×, byte-identical
|
||||
determinism, constrained-hardware envelope).
|
||||
|
||||
## 2. What the kernel is — and the boundary that makes it safe
|
||||
|
||||
One napi-rs crate (`codegraph-kernel`) linking tree-sitter's C library and native grammars.
|
||||
Input `(filePath, content, language)` per file; output **flat typed buffers** (nodes, edges,
|
||||
unresolved refs) — one boundary crossing per file. It replaces ONLY the parse+extract walk
|
||||
inside the parse workers, behind the existing `ExtractionResult` contract.
|
||||
|
||||
**Never ported (works unchanged for all languages from day one):** name-matcher +
|
||||
import-resolver, all framework resolvers (`src/resolution/frameworks/`), all 36 synthesis
|
||||
passes, MCP/explore, sync/watcher, installer. They consume the graph and raw source, not
|
||||
the parse tree.
|
||||
|
||||
**Coexistence is permanent:** a language routes to the kernel only after its gate passes;
|
||||
everything else stays on the wasm path forever if need be. No flag-day. Rollback per
|
||||
language = flipping the route.
|
||||
|
||||
**Distribution:** prebuilt `.node` per platform through the existing release-bundle
|
||||
pipeline (`scripts/build-bundle.sh` + per-platform npm packages); the same crate compiled
|
||||
to wasm is the universal fallback. Zero-native-build-on-install stays true.
|
||||
|
||||
## 3. Phase 0 — scaffold (do first, ~days)
|
||||
|
||||
1. `codegraph-kernel/` crate: napi-rs, tree-sitter C, rayon optional (workers already
|
||||
parallelize per-file — start synchronous per call, one kernel call per file from the
|
||||
existing `ParseWorkerPool` workers; do NOT rebuild the pool).
|
||||
2. Buffer contract: decide the flat encoding (suggest: one `Buffer` per table,
|
||||
fixed-width rows + a string arena; version byte first). Write the TS decoder next to
|
||||
`parse-worker.ts`.
|
||||
3. Generic emitter driven by per-language `.scm` query files + a small per-language Rust
|
||||
config (node-kind → NodeKind mapping, name-field conventions). Escape hatch: a
|
||||
per-language `post(buffers, source)` TS hook for logic queries can't express.
|
||||
4. Build integration: napi prebuilds wired into the release workflow next to the Node
|
||||
bundles; `CODEGRAPH_KERNEL=0` kill switch; wasm fallback auto-selected when the
|
||||
`.node` is absent (source runs, unsupported platforms).
|
||||
5. CI: assert native grammars and wasm grammars are built from the SAME grammar source
|
||||
revisions (ABI drift between paths would make per-language routing non-deterministic).
|
||||
|
||||
### 3a. Phase 0 — SHIPPED 2026-07-16 (what exists and the decisions made)
|
||||
|
||||
- **Crate:** `codegraph-kernel/` (napi 3, tree-sitter 0.25, no CLI dependency —
|
||||
`scripts/build-kernel.sh` does cargo build + stage into
|
||||
`codegraph-kernel/prebuilds/<platform>-<arch>/codegraph-kernel.node`; `npm run
|
||||
build:kernel`). Exports `extractFile`, `contractInfo`, `grammarInfo`.
|
||||
- **Buffer contract v1:** five Buffers (meta/nodes/edges/refs/arena), fixed-width LE rows,
|
||||
string arena with `(offset,len)` refs, `0xFFFFFFFF` = absent, version byte first, node
|
||||
IDs computed Rust-side (sha256, byte-identical to `generateNodeId` — pinned by test),
|
||||
tri-state bool flags, `extraJson` escape slot per node row, and a RESERVED u32 metrics
|
||||
slot (Arc 3.2). Layout doc lives twice and must match: `codegraph-kernel/src/buffers.rs`
|
||||
↔ `src/extraction/kernel/layout.ts`. NODE_KINDS/EDGE_KINDS array ORDER in src/types.ts
|
||||
is wire contract now (EDGE_KINDS became a runtime array for this).
|
||||
- **Emitter:** generic, `.scm`-driven (`@def.<NodeKind>` + `@name` + `@ref.<EdgeKind>`
|
||||
capture convention), scope stack by byte-range nesting → `::`-joined qualifiedNames,
|
||||
contains edges, refs attached to innermost enclosing def (file node fallback) — the
|
||||
TreeSitterExtractor conventions. Seed TS/JS queries are SMOKE-level only; R2 replaces.
|
||||
- **Routing:** inside `extractFromSource` (tree-sitter.ts) — `tryKernelExtract` first,
|
||||
wasm `TreeSitterExtractor` as fallback (also per-FILE fallback on any kernel error).
|
||||
DEFAULT_ROUTED is EMPTY; dev opt-in via `CODEGRAPH_KERNEL_LANGS=<langs|all>`; global
|
||||
kill switch `CODEGRAPH_KERNEL=0`; loader verifies ABI + kind tables before routing
|
||||
(stale .node → silent wasm, `CODEGRAPH_KERNEL_DEBUG=1` to see why). The escape hatch
|
||||
landed as `post(result, source)` over the DECODED result (not raw buffers) — decoded
|
||||
is what TS logic wants; see POST_PASSES in `src/extraction/kernel/index.ts`.
|
||||
- **Grammar parity (the §3.5 CI) — and a decision that changed the wasm path:** the
|
||||
parity test (`__tests__/kernel-grammar-parity.test.ts`, behavioral: ABI + node-kind +
|
||||
field tables compared id-by-id) caught on day one that tree-sitter-wasms ships
|
||||
2023-era TS/JS grammars (^0.20.x) vs crates.io current. Resolution: **vendored fresh
|
||||
wasm into `src/extraction/wasm/` built from the exact crate revisions** —
|
||||
tree-sitter-typescript v0.23.2 (f975a62) for typescript+tsx, tree-sitter-javascript
|
||||
v0.25.0 (44c892e) for javascript+jsx — from each repo's CHECKED-IN parser.c (no
|
||||
`generate`), tree-sitter-cli 0.25.10, emcc. So the production wasm TS/JS grammars are
|
||||
UPGRADED as of this change (full suite green, 2456 tests) and **R2/R3 parity diffs
|
||||
are grammar-neutral**. Bump crate + vendored wasm together, or the parity test fails.
|
||||
- **Release wiring:** `kernel` matrix job in release.yml (macos-14 ×2 targets,
|
||||
ubuntu-22.04, ubuntu-22.04-arm, windows-latest ×2 — all continue-on-error: kernel is
|
||||
optional, a toolchain flake never blocks a release) → artifacts → `release/kernel/` →
|
||||
build-bundle.sh stages `lib/kernel/codegraph-kernel.node` when present. The release
|
||||
job runs the kernel tests with `CODEGRAPH_KERNEL_EXPECT=1` (missing binary = FAILURE
|
||||
there, skip elsewhere).
|
||||
- **Loader search order:** `CODEGRAPH_KERNEL_PATH` → `<pkgroot>/kernel/` (bundle) →
|
||||
`<pkgroot>/codegraph-kernel/prebuilds/<plat>-<arch>/` (source runs).
|
||||
- **Known R2 gate item:** native columns are UTF-8 byte offsets; web-tree-sitter's are
|
||||
UTF-16-derived — column NUMBERS on non-ASCII lines will differ in parity dumps
|
||||
(text, lines, IDs unaffected). Classify or normalize when it shows up.
|
||||
|
||||
## 4. Per-language tracker
|
||||
|
||||
Tiers: **T1** = mostly `.scm` + mapping config. **T2** = needs bespoke pre/post passes kept
|
||||
in TS (listed). **T3** = not a plain tree-sitter walk (standalone/multi-grammar extractor)
|
||||
— migrate last or never; wasm/TS path is a fine permanent home.
|
||||
|
||||
The user-facing language contract is `README.md → Language Support` (34 logos incl.
|
||||
Metal, CUDA, Terraform/OpenTofu, Pascal/Delphi). Keep this tracker in sync with it —
|
||||
every README language must have a row here, even the ones that only ride another
|
||||
language's port.
|
||||
|
||||
Grammar column: `crates.io` = mainstream native grammar crate exists; `vendored` = we ship
|
||||
a rebuilt/patched wasm (ABI-15) and the kernel must compile OUR fork natively — verify
|
||||
parity before porting the language.
|
||||
|
||||
| Language(s) | Today | Tier | Grammar source | Migration notes / known traps | Status |
|
||||
|---|---|---|---|---|---|
|
||||
| typescript, tsx, javascript, jsx | `languages/typescript.ts`, `javascript.ts` + shared branches | T1 | crates.io | First target. Value-reference edges (#895/#897) and component recognition (#841 forwardRef/memo/styled) must survive — they're extraction-side. Largest test surface; gate is strictest here. | ☐ |
|
||||
| java | `languages/java.ts` | T1 | crates.io | Second target; unlocks the dubbo-parity claim. Lombok member synthesis (#912) is a NODE synthesizer hook in extraction (`synthesizeMembers`) — port or keep as TS post-pass. | ☐ |
|
||||
| python | `languages/python.ts` | T1 | crates.io | Third. Decorator extraction feeds framework route detection — parity required. | ☐ |
|
||||
| go | `languages/go.ts` | T1 | crates.io | Third (tie). Value-reference edges ship here too (#897). | ☐ |
|
||||
| ruby, php | dedicated files | T1 | crates.io | Straightforward; PHP property-receiver shapes (#1220/#1251) are RESOLUTION-side, unaffected. | ☐ |
|
||||
| csharp | `languages/csharp.ts` | T1 | crates.io | Plain. | ☐ |
|
||||
| rust, dart, scala, lua, luau, r | dedicated files | T1 | crates.io (luau/r/scala: verify crate freshness vs our wasm) | Long-tail T1; port opportunistically after the big five. | ☐ |
|
||||
| kotlin | `languages/kotlin.ts` | T1½ | crates.io | Expect/actual pairing is synthesis-side (fine); extraction is clean but validate against a KMP repo. | ☐ |
|
||||
| swift | shared + dedicated branch | T1½ | crates.io | **Trap:** in-class property extraction lives in `tree-sitter.ts`'s DEDICATED branch, not `swift.ts` (#1020 — Alamofire went 0→348 props). Gate on Alamofire. | ☐ |
|
||||
| c, cpp | `languages/c-cpp.ts` | **T2** | crates.io | Keep as TS pre-passes: `blankCppExportMacros`/`blankCppInlineMacros` (UE `class MACRO Name` phantom-function misparse, #1096–#1102, CARLA 440→6), in-body reflection collapse guard (#1206), content-based `.h` C-vs-C++ detection. | ☐ |
|
||||
| metal, cuda | dialects over the cpp grammar | **T2** (rides c/cpp) | crates.io (cpp) | README-listed as first-class languages. Both are dialect-gated cpp: Metal = specifier/`[[attribute]]` blanking (#1121, the preParse-takes-filePath pattern); CUDA = `<<<>>>` blanking + content-gated `.h` (#1172). Their pre-passes must run before the kernel parse or stay TS-side; gate them WITH the c/cpp port, not separately. | ☐ |
|
||||
| objc | `languages/objc.ts` | T2 | crates.io | Rides the c-cpp trap family; RN bridge extraction feeds `rnCrossPlatformEdges` (synthesis-side, fine). | ☐ |
|
||||
| arkts | `languages/arkts.ts` | T2 | **vendored** (harmony-contrib) | Dot-prefixed refs + decorator-gated matching fixed 36,840 wrong edges — that logic must port exactly or stay TS-side. Compile our grammar fork natively. | ☐ |
|
||||
| pascal | `languages/pascal.ts` | T2 | **vendored** | Paired with dfm-extractor (T3); `extractPascalDefProc` indexed lookups. | ☐ |
|
||||
| vbnet | `languages/vbnet.ts` | T2 | **vendored, patched + external scanner** | Our wasm is a patched grammar WITH a C external scanner — the kernel must build that scanner; ts-cli 0.24 dropped `\p{...}` classes during the original build (#1164). Highest grammar-build risk of any language. | ☐ |
|
||||
| cobol | `languages/cobol.ts` | T2 | **vendored fork** | Paragraph-extent reconstruction + copybook resolution are extraction logic (#1161, CardDemo 43/44). Port carefully or keep TS post-pass. | ☐ |
|
||||
| erlang | `languages/erlang.ts` | T2 | **vendored (WhatsApp/ELP)** | npm `tree-sitter-erlang` is HIJACKED — never source from it (#1165). gen_server dispatch is synthesis-side (fine). | ☐ |
|
||||
| nix | `languages/nix.ts` | T2 | **vendored (ABI-15 rebuild)** | Option-path synthesizer is synthesis-side; the `===`-always-false → `.equals()` lesson (#1190) is wasm-binding-specific and disappears natively — still gate on nixpkgs (44k files). | ☐ |
|
||||
| solidity | `languages/solidity.ts` | T2 | **vendored** | `modifier_invocation` outside body walk (#1170) is extraction-side; port it. | ☐ |
|
||||
| terraform | `languages/terraform.ts` | T2 | **vendored** | `:`-scoped refs for module-boundary bridging (#1173); metadata does NOT persist — re-read source (#1174). | ☐ |
|
||||
| cfml, cfscript, cfquery | `cfml-extractor.ts` + 3 grammar files | **T3** | **vendored ×3** | 3-grammar family with BOM-sensitive dialect sniffing (#1118/#1153–55). Leave on wasm until the very end, possibly forever. | ☐ |
|
||||
| svelte, vue, astro, liquid | standalone extractors | **T3** | n/a (custom/embedded parsing) | Not tree-sitter walks. Permanent TS home is acceptable — file counts are small and these repos are small. | ☐ |
|
||||
| dfm (Delphi forms), razor, mybatis XML | standalone extractors | **T3** | n/a | Same as above. mybatis pairs with a synthesis pass (fine). | ☐ |
|
||||
|
||||
**Do-not-regress invariants during any port** (extraction-side, will show up in the gate):
|
||||
node metadata is re-read from source, never persisted; parse commits stay in FILE ORDER
|
||||
(#1015); `MAX_FILE_SIZE` skip; generated-file detection; `CODEGRAPH_PARSE_WORKERS`
|
||||
semantics; framework `extract()` hooks keep running TS-side per file after the kernel pass.
|
||||
|
||||
## 5. Equivalence gate (run per language, no exceptions)
|
||||
|
||||
Byte-identity vs hand-written extractors is NOT expected — the gate is behavioral parity:
|
||||
|
||||
1. **Graph parity:** fresh-index 3 real repos (small/medium/large for the language) on
|
||||
wasm-path vs kernel-path builds. Dump with the `dump-graph.mjs` pattern (natural keys).
|
||||
Node/edge/ref deltas ≤0.5% AND every diff category manually classified (the 13-edge
|
||||
supertype-visibility bug this week was caught exactly this way — small diffs are real).
|
||||
2. **Retrieval invariants:** the language's canonical flows still connect end-to-end in
|
||||
`codegraph_explore` (playbook: `docs/design/dynamic-dispatch-coverage-playbook.md`);
|
||||
node counts stable; synthesized-edge spot-check.
|
||||
3. **Agent A/B non-regression** per the standard methodology (CLAUDE.md): `--model sonnet
|
||||
--effort high` ALWAYS, ≥2 runs/arm, pre-warmed daemon, `CODEGRAPH_NO_PROMPT_HOOK=1`,
|
||||
forbid subagent delegation in the prompt.
|
||||
4. **Perf:** fresh-index improves on the language's repos; a NON-migrated control repo is
|
||||
unchanged; suite green; Linux docker + Windows VM passes for platform-sensitive bits.
|
||||
|
||||
## 6. Rollout order and expected wins
|
||||
|
||||
1. **TS/JS/TSX/JSX** — most indexed files in the funnel; excalidraw 3.3s → ~2.3s expected.
|
||||
2. **Java** — dubbo 11.1s → ~7.5s expected (**the cbm-parity headline**).
|
||||
3. **Python, Go** — rounds out ~90% of real-world indexed files.
|
||||
4. Kernel-scale re-run in the cg1212 container after (2): parse 6.0m → ~1.5–2m expected.
|
||||
5. Long tail opportunistically; T3 possibly never — that's fine by design.
|
||||
|
||||
Measurement discipline (hard-won this week — do NOT relearn these):
|
||||
- Profile first. Ideas killed by measurement this week: sorted-chunk inserts (zero),
|
||||
statement-batching the persist (zero — B-tree maintenance is the cost), RAM-disk/
|
||||
in-memory DB build (SLOWER — fastInit already writes at page-cache speed).
|
||||
- `CODEGRAPH_SYNTH_TIMINGS=1` now emits full phase walls (`[phase-timing]`) + pool/batch
|
||||
timings. UI distorts phase walls — pipe stdout away.
|
||||
- Check host load before timing (iOS simulators inflated every phase ~30%); the
|
||||
Monitor-on-loadavg pattern (fire <3.5) gives clean windows.
|
||||
- `grep` is aliased to ugrep and silently treats `callback-synthesizer.ts` as binary —
|
||||
use `grep -a`.
|
||||
|
||||
## 7. AFTER the kernel: the follow-on roadmap (in order)
|
||||
|
||||
### 7a. Kernel-scale resolution speed
|
||||
The kernel makes parse fast; at Linux-kernel scale resolution is now the wall (19.5min
|
||||
sequential in the 2-CPU container; the resolver pool requires ≥4 cores to engage).
|
||||
Steps: re-run cg1212 validation on ≥4-core allocation (pool + parallel synthesis engage);
|
||||
profile; likely levers: worker count scaling, batch size at scale, `warmCachesYielding`
|
||||
on multi-GB DBs. Target: kernel <10min on a normal 8-core host.
|
||||
|
||||
### 7b. Arc 3 — graph richness (forensics-backed; adopt cbm's real extras, skip inflation)
|
||||
Priority order, each gated by the standard A/B + node-explosion probes:
|
||||
1. **Test→subject edges** (first-class `tests` edges at index time; we compute covering
|
||||
tests at query time today; cbm materializes 14.8k on dubbo). Feeds test-gap detection
|
||||
(Lite headline) + Pro risk signals. Cheapest, do first.
|
||||
2. **Per-node code metrics** (complexity, cognitive, `is_test`, `is_entry_point`,
|
||||
param counts) — computed during extraction (the kernel makes this nearly free —
|
||||
design the buffer contract with a metrics slot!). Feeds Pro risk-ranking verdicts +
|
||||
explore ranking de-noise.
|
||||
3. **Read/write distinction on references** (`USAGE` vs `WRITES`). The measured agent
|
||||
frontier ("who mutates this state" — the canvasNonce class). HIGHEST value, HIGHEST
|
||||
risk: scope to exported/state-relevant symbols; the tracking-every-local explosion is
|
||||
the known failure mode (#999/#1212 class). Full validation methodology.
|
||||
4. **Exception-flow edges** (`raises`) — throw→handler; moderate.
|
||||
5. **Doc Section nodes** (markdown headings as nodes, linked to code) — maps onto Pro's
|
||||
synced-business-docs story.
|
||||
6. **IaC nodes** (k8s/docker/kustomize as graph nodes with cross-references).
|
||||
NOT worth chasing (verified in their cache schema): per-variable node inflation (85% of
|
||||
their node count), DB size parity (theirs is ~60% allocation slack), similarity vectors
|
||||
in the core engine.
|
||||
|
||||
### 7c. Deferred/parked (needs explicit approval before starting)
|
||||
- Single-file SEA binary (distribution polish; zero speed).
|
||||
- Team-shared graph artifact (cbm's `graph.db.zst` idea — good, but design it for the
|
||||
Pro shared-worker story, not as an OSS clone).
|
||||
- Full native rewrite: rejected with data — the moat (2,444 tests, byte-identical
|
||||
determinism, this week's two caught-by-gate bugs) lives in the TS reference.
|
||||
|
||||
## 8. Context for the executing agent
|
||||
|
||||
- House rules live in `CLAUDE.md` (repo root) — the retrieval invariants, A/B model
|
||||
policy, release rules (never `npm publish`/push tags), changelog format.
|
||||
- This week's PR trail tells the story and the style: #1305, #1320 (checkpoint deferral +
|
||||
double-buffered persist; THE invariant: batch k+1 READS batch k's edges — supertype
|
||||
walks — so edges insert before fan-out), #1321 (parallel synthesis via pool reuse,
|
||||
registry order = merge order), #1322 (bulk edge load, identity index stays), #1323
|
||||
(kernel-scale hardening: skip-don't-retry-on-main >1.5M nodes, yielding index recreate).
|
||||
- Every perf PR shipped byte-identical with the dump-diff gate; keep that bar.
|
||||
- Competitive context (validated 2026-07-16): cbm wins medium-repo fresh index 1.55–1.8×
|
||||
(their RAM-first design); we win sync 2.4–2.8×, agent A/B (their 14 tools drew ZERO
|
||||
calls in 8/8 runs), call-graph density 1.3–2.3×, and the constrained-hardware envelope
|
||||
(Linux kernel on 2-CPU/6GB: we complete in 27min, they die at 0.16% — their speed IS
|
||||
their memory floor). The kernel project closes their last number without giving up any
|
||||
of ours.
|
||||
@@ -21,6 +21,7 @@
|
||||
"preuninstall": "node dist/bin/uninstall.js",
|
||||
"copy-assets": "node -e \"const fs=require('fs');fs.mkdirSync('dist/db',{recursive:true});fs.copyFileSync('src/db/schema.sql','dist/db/schema.sql');fs.mkdirSync('dist/extraction/wasm',{recursive:true});fs.readdirSync('src/extraction/wasm').filter(f=>f.endsWith('.wasm')).forEach(f=>fs.copyFileSync('src/extraction/wasm/'+f,'dist/extraction/wasm/'+f))\"",
|
||||
"dev": "tsc --watch",
|
||||
"build:kernel": "bash scripts/build-kernel.sh",
|
||||
"cli": "npm run build && node dist/bin/codegraph.js",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
|
||||
@@ -68,6 +68,26 @@ echo "[bundle] installing production dependencies"
|
||||
( cd "$STAGE/lib" && npm ci --omit=dev --ignore-scripts >/dev/null 2>&1 )
|
||||
rm -f "$STAGE/lib/package-lock.json"
|
||||
|
||||
# 3b. Native extraction kernel (optional). Included when a prebuilt .node for
|
||||
# the target exists — release/kernel/<target>/codegraph-kernel.node (the
|
||||
# release workflow's prebuild artifacts) or the locally staged
|
||||
# codegraph-kernel/prebuilds/<target>/ (scripts/build-kernel.sh). Absent →
|
||||
# the bundle simply runs the wasm extraction path; the kernel is a
|
||||
# per-language speedup, never a requirement (see
|
||||
# docs/design/rust-kernel-migration-plan.md).
|
||||
KERNEL_NODE=""
|
||||
for candidate in "$ROOT/release/kernel/${TARGET}/codegraph-kernel.node" \
|
||||
"$ROOT/codegraph-kernel/prebuilds/${TARGET}/codegraph-kernel.node"; do
|
||||
if [ -f "$candidate" ]; then KERNEL_NODE="$candidate"; break; fi
|
||||
done
|
||||
if [ -n "$KERNEL_NODE" ]; then
|
||||
mkdir -p "$STAGE/lib/kernel"
|
||||
cp "$KERNEL_NODE" "$STAGE/lib/kernel/codegraph-kernel.node"
|
||||
echo "[bundle] native kernel included ($KERNEL_NODE)"
|
||||
else
|
||||
echo "[bundle] no native kernel for ${TARGET} — bundle uses the wasm extraction path"
|
||||
fi
|
||||
|
||||
# 4. Vendored Node + launcher (the launcher uses the bundled Node by relative
|
||||
# path, so no system Node is ever needed).
|
||||
#
|
||||
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build the native extraction kernel (codegraph-kernel) and stage the .node
|
||||
# where the TS loader (src/extraction/kernel/loader.ts) finds it for
|
||||
# from-source runs and tests:
|
||||
#
|
||||
# codegraph-kernel/prebuilds/<platform>-<arch>/codegraph-kernel.node
|
||||
#
|
||||
# The kernel is OPTIONAL everywhere: when the .node is absent the extraction
|
||||
# path falls back to the wasm pipeline. This script needs a Rust toolchain
|
||||
# (rustup.rs); nothing else in the repo does.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/build-kernel.sh # host platform
|
||||
# scripts/build-kernel.sh --target <rust-triple> [--platform <plat-arch>]
|
||||
#
|
||||
# The cross-compile form is what the release workflow uses (e.g.
|
||||
# --target x86_64-apple-darwin --platform darwin-x64 on a macos-arm runner).
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
CRATE="$ROOT/codegraph-kernel"
|
||||
|
||||
TARGET=""
|
||||
PLATFORM=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--target) TARGET="$2"; shift 2 ;;
|
||||
--platform) PLATFORM="$2"; shift 2 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Map a rust triple (or the host) to the bundle-target naming used across the
|
||||
# release pipeline (darwin-arm64, linux-x64, win32-arm64, ...).
|
||||
if [ -z "$PLATFORM" ]; then
|
||||
if [ -n "$TARGET" ]; then
|
||||
case "$TARGET" in
|
||||
aarch64-apple-darwin) PLATFORM="darwin-arm64" ;;
|
||||
x86_64-apple-darwin) PLATFORM="darwin-x64" ;;
|
||||
x86_64-unknown-linux-gnu) PLATFORM="linux-x64" ;;
|
||||
aarch64-unknown-linux-gnu) PLATFORM="linux-arm64" ;;
|
||||
x86_64-pc-windows-msvc) PLATFORM="win32-x64" ;;
|
||||
aarch64-pc-windows-msvc) PLATFORM="win32-arm64" ;;
|
||||
*) echo "cannot map rust target '$TARGET' to a platform name; pass --platform" >&2; exit 1 ;;
|
||||
esac
|
||||
else
|
||||
case "$(uname -s)-$(uname -m)" in
|
||||
Darwin-arm64) PLATFORM="darwin-arm64" ;;
|
||||
Darwin-x86_64) PLATFORM="darwin-x64" ;;
|
||||
Linux-x86_64) PLATFORM="linux-x64" ;;
|
||||
Linux-aarch64) PLATFORM="linux-arm64" ;;
|
||||
MINGW*-x86_64|MSYS*-x86_64) PLATFORM="win32-x64" ;;
|
||||
MINGW*-aarch64|MSYS*-aarch64) PLATFORM="win32-arm64" ;;
|
||||
*) echo "unrecognized host $(uname -s)-$(uname -m); pass --platform" >&2; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[kernel] building codegraph-kernel for ${PLATFORM}${TARGET:+ (target $TARGET)}"
|
||||
cd "$CRATE"
|
||||
if [ -n "$TARGET" ]; then
|
||||
rustup target add "$TARGET" >/dev/null 2>&1 || true
|
||||
cargo build --release --target "$TARGET"
|
||||
OUTDIR="$CRATE/target/$TARGET/release"
|
||||
else
|
||||
cargo build --release
|
||||
OUTDIR="$CRATE/target/release"
|
||||
fi
|
||||
|
||||
# cdylib name differs per OS; the staged name is always codegraph-kernel.node.
|
||||
case "$PLATFORM" in
|
||||
darwin-*) LIB="$OUTDIR/libcodegraph_kernel.dylib" ;;
|
||||
linux-*) LIB="$OUTDIR/libcodegraph_kernel.so" ;;
|
||||
win32-*) LIB="$OUTDIR/codegraph_kernel.dll" ;;
|
||||
esac
|
||||
[ -f "$LIB" ] || { echo "[kernel] error: built library not found at $LIB" >&2; exit 1; }
|
||||
|
||||
DEST="$CRATE/prebuilds/$PLATFORM"
|
||||
mkdir -p "$DEST"
|
||||
cp "$LIB" "$DEST/codegraph-kernel.node"
|
||||
echo "[kernel] staged $DEST/codegraph-kernel.node ($(du -h "$DEST/codegraph-kernel.node" | cut -f1))"
|
||||
@@ -271,10 +271,23 @@ export async function initGrammars(): Promise<void> {
|
||||
* nix-community/tree-sitter-nix @ 3d0173d (MIT) with tree-sitter-cli 0.25.10
|
||||
* (`generate` + `build --wasm`, ABI 15 — upstream's checked-in parser.c is
|
||||
* still ABI 13; all 54 upstream corpus tests pass on the regenerated parser).
|
||||
*
|
||||
* TypeScript/TSX/JavaScript (+jsx, which shares the javascript grammar): the
|
||||
* tree-sitter-wasms builds are 2023-era (^0.20.x); we vendor wasm built from
|
||||
* the SAME grammar revisions the native extraction kernel compiles
|
||||
* (codegraph-kernel/Cargo.toml), so the kernel path and the wasm fallback
|
||||
* parse identically and per-language routing stays graph-neutral:
|
||||
* - tree-sitter/tree-sitter-typescript v0.23.2 (f975a62) → typescript + tsx
|
||||
* - tree-sitter/tree-sitter-javascript v0.25.0 (44c892e) → javascript + jsx
|
||||
* Built from each repo's CHECKED-IN parser.c (no `generate`) with
|
||||
* tree-sitter-cli 0.25.10 `build --wasm` — the same tables crates.io compiles.
|
||||
* The kernel-grammar-parity test asserts this alignment; bump the crate and
|
||||
* the vendored wasm together.
|
||||
*/
|
||||
const VENDORED_WASM_LANGS: ReadonlySet<GrammarLanguage> = new Set([
|
||||
'pascal', 'scala', 'lua', 'luau', 'csharp', 'r', 'cfml', 'cfscript', 'cfquery',
|
||||
'cobol', 'vbnet', 'erlang', 'terraform', 'arkts', 'nix',
|
||||
'typescript', 'tsx', 'javascript', 'jsx',
|
||||
]);
|
||||
|
||||
/** Absolute path of a language's grammar WASM (vendored or tree-sitter-wasms). */
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Decode the kernel's flat buffers into an ExtractionResult — the single
|
||||
* JS-side pass over the per-file tables. See layout.ts for the byte layout
|
||||
* and codegraph-kernel/src/buffers.rs for the writer.
|
||||
*/
|
||||
|
||||
import type {
|
||||
Edge,
|
||||
EdgeKind,
|
||||
ExtractionError,
|
||||
ExtractionResult,
|
||||
Language,
|
||||
Node,
|
||||
NodeKind,
|
||||
ReferenceKind,
|
||||
UnresolvedReference,
|
||||
} from '../../types';
|
||||
import { NODE_KINDS, EDGE_KINDS } from '../../types';
|
||||
import type { KernelBuffers } from './loader';
|
||||
import {
|
||||
EDGE,
|
||||
EDGE_ROW_SIZE,
|
||||
FLAG,
|
||||
FUNCTION_REF_CODE,
|
||||
KERNEL_ABI_VERSION,
|
||||
META,
|
||||
META_SIZE,
|
||||
NODE,
|
||||
NODE_ROW_SIZE,
|
||||
NONE,
|
||||
PROVENANCES,
|
||||
REF,
|
||||
REF_ROW_SIZE,
|
||||
VISIBILITIES,
|
||||
} from './layout';
|
||||
|
||||
/** Read an (offset, len) arena string; undefined when absent. */
|
||||
function str(arena: Buffer, row: Buffer, at: number): string | undefined {
|
||||
const off = row.readUInt32LE(at);
|
||||
if (off === NONE) return undefined;
|
||||
const len = row.readUInt32LE(at + 4);
|
||||
return arena.toString('utf8', off, off + len);
|
||||
}
|
||||
|
||||
/** NUL-joined list field; undefined when absent. */
|
||||
function strList(arena: Buffer, row: Buffer, at: number): string[] | undefined {
|
||||
const joined = str(arena, row, at);
|
||||
return joined === undefined ? undefined : joined.split('\0');
|
||||
}
|
||||
|
||||
/** Tri-state boolean from a (present, value) bit pair. */
|
||||
function flag(flags: number, pair: number): boolean | undefined {
|
||||
if ((flags & (1 << (pair * 2))) === 0) return undefined;
|
||||
return (flags & (1 << (pair * 2 + 1))) !== 0;
|
||||
}
|
||||
|
||||
function u32opt(row: Buffer, at: number): number | undefined {
|
||||
const v = row.readUInt32LE(at);
|
||||
return v === NONE ? undefined : v;
|
||||
}
|
||||
|
||||
export function decodeExtractBuffers(
|
||||
buffers: KernelBuffers,
|
||||
filePath: string,
|
||||
language: Language
|
||||
): ExtractionResult {
|
||||
const { meta, arena } = buffers;
|
||||
if (meta.length < META_SIZE) throw new Error(`kernel meta too short: ${meta.length}`);
|
||||
const version = meta.readUInt8(META.version);
|
||||
if (version !== KERNEL_ABI_VERSION) {
|
||||
throw new Error(`kernel buffer ABI ${version} != expected ${KERNEL_ABI_VERSION}`);
|
||||
}
|
||||
const nodeCount = meta.readUInt32LE(META.nodeCount);
|
||||
const edgeCount = meta.readUInt32LE(META.edgeCount);
|
||||
const refCount = meta.readUInt32LE(META.refCount);
|
||||
|
||||
const now = Date.now();
|
||||
const nodes: Node[] = new Array(nodeCount);
|
||||
// Node-table row index → node id, for edge/ref endpoint resolution.
|
||||
const idByRow: string[] = new Array(nodeCount);
|
||||
|
||||
for (let i = 0; i < nodeCount; i++) {
|
||||
const row = buffers.nodes.subarray(i * NODE_ROW_SIZE, (i + 1) * NODE_ROW_SIZE);
|
||||
const id = str(arena, row, NODE.id)!;
|
||||
idByRow[i] = id;
|
||||
const flags = row.readUInt16LE(NODE.flags);
|
||||
const node: Node = {
|
||||
id,
|
||||
kind: NODE_KINDS[row.readUInt8(NODE.kind)] as NodeKind,
|
||||
name: str(arena, row, NODE.name)!,
|
||||
qualifiedName: str(arena, row, NODE.qualifiedName)!,
|
||||
filePath,
|
||||
language,
|
||||
startLine: row.readUInt32LE(NODE.startLine),
|
||||
endLine: row.readUInt32LE(NODE.endLine),
|
||||
startColumn: row.readUInt32LE(NODE.startColumn),
|
||||
endColumn: row.readUInt32LE(NODE.endColumn),
|
||||
updatedAt: now,
|
||||
};
|
||||
const docstring = str(arena, row, NODE.docstring);
|
||||
if (docstring !== undefined) node.docstring = docstring;
|
||||
const signature = str(arena, row, NODE.signature);
|
||||
if (signature !== undefined) node.signature = signature;
|
||||
const visibility = VISIBILITIES[row.readUInt8(NODE.visibility)];
|
||||
if (visibility !== undefined) node.visibility = visibility;
|
||||
const isExported = flag(flags, FLAG.isExported);
|
||||
if (isExported !== undefined) node.isExported = isExported;
|
||||
const isAsync = flag(flags, FLAG.isAsync);
|
||||
if (isAsync !== undefined) node.isAsync = isAsync;
|
||||
const isStatic = flag(flags, FLAG.isStatic);
|
||||
if (isStatic !== undefined) node.isStatic = isStatic;
|
||||
const isAbstract = flag(flags, FLAG.isAbstract);
|
||||
if (isAbstract !== undefined) node.isAbstract = isAbstract;
|
||||
const decorators = strList(arena, row, NODE.decorators);
|
||||
if (decorators !== undefined) node.decorators = decorators;
|
||||
const typeParameters = strList(arena, row, NODE.typeParameters);
|
||||
if (typeParameters !== undefined) node.typeParameters = typeParameters;
|
||||
const returnType = str(arena, row, NODE.returnType);
|
||||
if (returnType !== undefined) node.returnType = returnType;
|
||||
const extraJson = str(arena, row, NODE.extraJson);
|
||||
if (extraJson !== undefined) Object.assign(node, JSON.parse(extraJson) as Partial<Node>);
|
||||
nodes[i] = node;
|
||||
}
|
||||
|
||||
const edges: Edge[] = new Array(edgeCount);
|
||||
for (let i = 0; i < edgeCount; i++) {
|
||||
const row = buffers.edges.subarray(i * EDGE_ROW_SIZE, (i + 1) * EDGE_ROW_SIZE);
|
||||
const sourceIdx = row.readUInt32LE(EDGE.sourceIdx);
|
||||
const targetIdx = row.readUInt32LE(EDGE.targetIdx);
|
||||
const edge: Edge = {
|
||||
source: sourceIdx === NONE ? str(arena, row, EDGE.sourceIdStr)! : idByRow[sourceIdx]!,
|
||||
target: targetIdx === NONE ? str(arena, row, EDGE.targetIdStr)! : idByRow[targetIdx]!,
|
||||
kind: EDGE_KINDS[row.readUInt8(EDGE.kind)] as EdgeKind,
|
||||
};
|
||||
const line = u32opt(row, EDGE.line);
|
||||
if (line !== undefined) edge.line = line;
|
||||
const column = u32opt(row, EDGE.column);
|
||||
if (column !== undefined) edge.column = column;
|
||||
const provenance = PROVENANCES[row.readUInt8(EDGE.provenance)];
|
||||
if (provenance !== undefined) edge.provenance = provenance;
|
||||
const metadataJson = str(arena, row, EDGE.metadataJson);
|
||||
if (metadataJson !== undefined) edge.metadata = JSON.parse(metadataJson) as Record<string, unknown>;
|
||||
edges[i] = edge;
|
||||
}
|
||||
|
||||
const unresolvedReferences: UnresolvedReference[] = new Array(refCount);
|
||||
for (let i = 0; i < refCount; i++) {
|
||||
const row = buffers.refs.subarray(i * REF_ROW_SIZE, (i + 1) * REF_ROW_SIZE);
|
||||
const fromIdx = row.readUInt32LE(REF.fromIdx);
|
||||
const kindByte = row.readUInt8(REF.kind);
|
||||
const ref: UnresolvedReference = {
|
||||
fromNodeId: fromIdx === NONE ? str(arena, row, REF.fromIdStr)! : idByRow[fromIdx]!,
|
||||
referenceName: str(arena, row, REF.referenceName)!,
|
||||
referenceKind:
|
||||
kindByte === FUNCTION_REF_CODE
|
||||
? 'function_ref'
|
||||
: (EDGE_KINDS[kindByte] as ReferenceKind),
|
||||
line: row.readUInt32LE(REF.line),
|
||||
column: row.readUInt32LE(REF.column),
|
||||
filePath,
|
||||
language,
|
||||
};
|
||||
const candidates = strList(arena, row, REF.candidates);
|
||||
if (candidates !== undefined) ref.candidates = candidates;
|
||||
unresolvedReferences[i] = ref;
|
||||
}
|
||||
|
||||
let errors: ExtractionError[] = [];
|
||||
const errorsOff = meta.readUInt32LE(META.errorsOff);
|
||||
if (errorsOff !== NONE) {
|
||||
const errorsLen = meta.readUInt32LE(META.errorsLen);
|
||||
errors = JSON.parse(arena.toString('utf8', errorsOff, errorsOff + errorsLen)) as ExtractionError[];
|
||||
}
|
||||
|
||||
return { nodes, edges, unresolvedReferences, errors, durationMs: 0 };
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* Kernel routing — which languages go through the native kernel, and the
|
||||
* single entry point the extraction path calls.
|
||||
*
|
||||
* Routing policy is deliberately TS-side and per-language (migration plan §2):
|
||||
* a language routes to the kernel only after its equivalence gate passes;
|
||||
* everything else stays on the wasm path forever if need be. Rollback per
|
||||
* language = removing it from DEFAULT_ROUTED (or CODEGRAPH_KERNEL=0 for all).
|
||||
*
|
||||
* R1 status: NO language is default-routed yet. Development/testing opt-in:
|
||||
* CODEGRAPH_KERNEL_LANGS=typescript,tsx (or "all" for every kernel-capable
|
||||
* language). R3 flips TS/JS into DEFAULT_ROUTED once the gate passes.
|
||||
*/
|
||||
|
||||
import type { ExtractionResult, Language } from '../../types';
|
||||
import { getKernel, kernelSupports } from './loader';
|
||||
import { decodeExtractBuffers } from './decode';
|
||||
|
||||
export { getKernel, kernelSupports, resetKernelForTests } from './loader';
|
||||
export { decodeExtractBuffers } from './decode';
|
||||
|
||||
/**
|
||||
* Languages routed to the kernel by default (gate-passed only — see the
|
||||
* per-language tracker in docs/design/rust-kernel-migration-plan.md §4).
|
||||
*/
|
||||
const DEFAULT_ROUTED: ReadonlySet<Language> = new Set<Language>([]);
|
||||
|
||||
/**
|
||||
* Per-language TS post-pass over the decoded result — the escape hatch for
|
||||
* logic `.scm` queries can't express (macro salvage, dialect sniffing,
|
||||
* wrapper-based component recognition). Runs synchronously after decode,
|
||||
* before the framework extract() hooks the caller applies. Keep these SMALL:
|
||||
* anything heavy belongs in the Rust emitter.
|
||||
*/
|
||||
export type KernelPostPass = (result: ExtractionResult, source: string) => void;
|
||||
const POST_PASSES: Partial<Record<Language, KernelPostPass>> = {
|
||||
// (none yet — R2+)
|
||||
};
|
||||
|
||||
function isRouted(language: Language): boolean {
|
||||
const env = process.env.CODEGRAPH_KERNEL_LANGS;
|
||||
if (env === undefined || env === '') return DEFAULT_ROUTED.has(language);
|
||||
if (env === 'all') return true;
|
||||
return env
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.includes(language);
|
||||
}
|
||||
|
||||
/** True when `language` would be extracted by the kernel right now. */
|
||||
export function kernelRoutes(language: Language): boolean {
|
||||
return isRouted(language) && kernelSupports(language);
|
||||
}
|
||||
|
||||
/** Warned-once registry so a broken language logs a single line, not one per file. */
|
||||
const warned = new Set<string>();
|
||||
|
||||
/**
|
||||
* Extract via the native kernel. Returns null when the kernel doesn't apply
|
||||
* (not routed / not available / kill switch) — the caller falls back to the
|
||||
* wasm TreeSitterExtractor. A kernel ERROR on a routed file also returns
|
||||
* null: per-file fallback keeps indexing correct while a kernel bug costs
|
||||
* only that file's speedup.
|
||||
*/
|
||||
export function tryKernelExtract(
|
||||
filePath: string,
|
||||
source: string,
|
||||
language: Language
|
||||
): ExtractionResult | null {
|
||||
if (!kernelRoutes(language)) return null;
|
||||
const kernel = getKernel();
|
||||
if (!kernel) return null;
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
// NOTE(T2 languages): when a preParse-carrying language (csharp #237,
|
||||
// metal #1121, cuda #1172, c/cpp macro blanking) routes here, its
|
||||
// offset-preserving preParse hook must be applied to `source` first —
|
||||
// wire that alongside the language's port, gated WITH its equivalence run.
|
||||
const buffers = kernel.extractFile(filePath, source, language);
|
||||
const result = decodeExtractBuffers(buffers, filePath, language);
|
||||
POST_PASSES[language]?.(result, source);
|
||||
result.durationMs = Date.now() - t0;
|
||||
return result;
|
||||
} catch (err) {
|
||||
if (!warned.has(language)) {
|
||||
warned.add(language);
|
||||
process.stderr.write(
|
||||
`[codegraph-kernel] ${language} extraction failed (${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}) — falling back to the wasm path\n`
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Native-kernel buffer layout — TS mirror of codegraph-kernel/src/buffers.rs.
|
||||
*
|
||||
* The kernel returns five Buffers per file: meta, nodes, edges, refs, arena.
|
||||
* Rows are fixed-width little-endian; strings are (offset, len) pairs into
|
||||
* the UTF-8 arena; `offset === NONE` means "field absent".
|
||||
*
|
||||
* THIS FILE AND buffers.rs MUST MATCH BYTE FOR BYTE. Any layout change bumps
|
||||
* KERNEL_ABI_VERSION on both sides — the loader refuses a version it doesn't
|
||||
* know and the extraction path falls back to wasm.
|
||||
*
|
||||
* NodeKind / EdgeKind / provenance / visibility cross the boundary as indexes
|
||||
* into NODE_KINDS / EDGE_KINDS (src/types.ts) and the small tables below, so
|
||||
* those array orders are part of the contract (append, never reorder). The
|
||||
* loader additionally verifies the kernel's own kind tables against
|
||||
* NODE_KINDS/EDGE_KINDS at load time, so a stale .node degrades to the wasm
|
||||
* path instead of mis-decoding.
|
||||
*/
|
||||
|
||||
export const KERNEL_ABI_VERSION = 1;
|
||||
|
||||
/** Sentinel for "absent" in u32 slots and string-ref offsets. */
|
||||
export const NONE = 0xffffffff;
|
||||
|
||||
export const META_SIZE = 36;
|
||||
export const NODE_ROW_SIZE = 96;
|
||||
export const EDGE_ROW_SIZE = 44;
|
||||
export const REF_ROW_SIZE = 40;
|
||||
|
||||
/** meta byte offsets */
|
||||
export const META = {
|
||||
version: 0, // u8
|
||||
nodeCount: 4, // u32
|
||||
edgeCount: 8, // u32
|
||||
refCount: 12, // u32
|
||||
arenaLen: 16, // u32
|
||||
errorsOff: 20, // u32 (NONE = no errors)
|
||||
errorsLen: 24, // u32
|
||||
durationMs: 28, // f64 (kernel-side wall; introspection only)
|
||||
} as const;
|
||||
|
||||
/** node row byte offsets */
|
||||
export const NODE = {
|
||||
kind: 0, // u8 — NODE_KINDS index
|
||||
visibility: 1, // u8 — VISIBILITIES index (0 = absent)
|
||||
flags: 2, // u16 — (present, value) bit pairs, see FLAG
|
||||
startLine: 4, // u32
|
||||
endLine: 8, // u32
|
||||
startColumn: 12, // u32
|
||||
endColumn: 16, // u32
|
||||
name: 20, // str
|
||||
qualifiedName: 28, // str
|
||||
id: 36, // str — kernel-computed node id
|
||||
docstring: 44, // str
|
||||
signature: 52, // str
|
||||
decorators: 60, // str — NUL-joined list
|
||||
typeParameters: 68, // str — NUL-joined list
|
||||
returnType: 76, // str
|
||||
extraJson: 84, // str — JSON of any extra Node props (escape hatch)
|
||||
metrics: 92, // u32 — reserved (Arc 3.2 per-node code metrics)
|
||||
} as const;
|
||||
|
||||
/** edge row byte offsets */
|
||||
export const EDGE = {
|
||||
sourceIdx: 0, // u32 (NONE → sourceIdStr)
|
||||
targetIdx: 4, // u32 (NONE → targetIdStr)
|
||||
kind: 8, // u8 — EDGE_KINDS index
|
||||
provenance: 9, // u8 — PROVENANCES index (0 = absent)
|
||||
line: 12, // u32 (NONE = absent)
|
||||
column: 16, // u32 (NONE = absent)
|
||||
metadataJson: 20, // str
|
||||
sourceIdStr: 28, // str
|
||||
targetIdStr: 36, // str
|
||||
} as const;
|
||||
|
||||
/** ref row byte offsets */
|
||||
export const REF = {
|
||||
fromIdx: 0, // u32 (NONE → fromIdStr)
|
||||
kind: 4, // u8 — EDGE_KINDS index, or FUNCTION_REF_CODE
|
||||
line: 8, // u32
|
||||
column: 12, // u32
|
||||
referenceName: 16, // str
|
||||
candidates: 24, // str — NUL-joined list
|
||||
fromIdStr: 32, // str
|
||||
} as const;
|
||||
|
||||
/** ReferenceKind wire code for the internal-only `function_ref` (#756). */
|
||||
export const FUNCTION_REF_CODE = 200;
|
||||
|
||||
/** Node bool-flag bit pairs: bit(2n) = present, bit(2n+1) = value. */
|
||||
export const FLAG = {
|
||||
isExported: 0,
|
||||
isAsync: 1,
|
||||
isStatic: 2,
|
||||
isAbstract: 3,
|
||||
} as const;
|
||||
|
||||
/** visibility byte values (0 = absent). */
|
||||
export const VISIBILITIES = [undefined, 'public', 'private', 'protected', 'internal'] as const;
|
||||
|
||||
/** provenance byte values (0 = absent). */
|
||||
export const PROVENANCES = [undefined, 'tree-sitter', 'scip', 'heuristic'] as const;
|
||||
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* Native-kernel loader — finds, loads, and contract-verifies the
|
||||
* codegraph-kernel .node addon.
|
||||
*
|
||||
* The kernel is OPTIONAL everywhere. Every failure mode here (no binary for
|
||||
* this platform, dlopen error, ABI/kind-table mismatch) resolves to `null`
|
||||
* and the extraction path silently keeps using the wasm pipeline — a missing
|
||||
* or stale kernel must never break indexing, only skip the speedup. Set
|
||||
* CODEGRAPH_KERNEL_DEBUG=1 to see why a kernel didn't load.
|
||||
*
|
||||
* Kill switch: CODEGRAPH_KERNEL=0 disables the kernel entirely (checked per
|
||||
* call so tests and embedders can flip it at runtime).
|
||||
*
|
||||
* Search order:
|
||||
* 1. CODEGRAPH_KERNEL_PATH — explicit .node path (dev/testing override)
|
||||
* 2. <up3>/kernel/codegraph-kernel.node — the release bundle layout
|
||||
* (lib/dist/** next to lib/kernel/; see scripts/build-bundle.sh)
|
||||
* 3. <up3>/codegraph-kernel/prebuilds/<platform>-<arch>/codegraph-kernel.node
|
||||
* — from-source runs and tests (staged by scripts/build-kernel.sh)
|
||||
*
|
||||
* "up3" = three directories above this file, which is the package root both
|
||||
* from src/extraction/kernel/ and from dist/extraction/kernel/.
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { createRequire } from 'module';
|
||||
import { NODE_KINDS, EDGE_KINDS } from '../../types';
|
||||
import { KERNEL_ABI_VERSION } from './layout';
|
||||
|
||||
/** Raw buffer tables for one file — see layout.ts for the byte layout. */
|
||||
export interface KernelBuffers {
|
||||
meta: Buffer;
|
||||
nodes: Buffer;
|
||||
edges: Buffer;
|
||||
refs: Buffer;
|
||||
arena: Buffer;
|
||||
}
|
||||
|
||||
export interface KernelContractInfo {
|
||||
abiVersion: number;
|
||||
kernelVersion: string;
|
||||
nodeKinds: string[];
|
||||
edgeKinds: string[];
|
||||
languages: string[];
|
||||
}
|
||||
|
||||
export interface KernelGrammarInfo {
|
||||
abiVersion: number;
|
||||
nodeKindCount: number;
|
||||
fieldCount: number;
|
||||
nodeKinds: string[];
|
||||
fieldNames: string[];
|
||||
}
|
||||
|
||||
export interface KernelModule {
|
||||
extractFile(filePath: string, content: string, language: string): KernelBuffers;
|
||||
contractInfo(): KernelContractInfo;
|
||||
grammarInfo(language: string): KernelGrammarInfo | null;
|
||||
}
|
||||
|
||||
const debugEnabled = () => process.env.CODEGRAPH_KERNEL_DEBUG === '1';
|
||||
function debug(msg: string): void {
|
||||
if (debugEnabled()) process.stderr.write(`[codegraph-kernel] ${msg}\n`);
|
||||
}
|
||||
|
||||
/** Languages the loaded binary supports (contract-verified). Empty when no kernel. */
|
||||
let kernelLanguages: ReadonlySet<string> = new Set();
|
||||
/** undefined = not attempted yet; null = attempted and unavailable. */
|
||||
let cached: KernelModule | null | undefined;
|
||||
|
||||
function candidatePaths(): string[] {
|
||||
const candidates: string[] = [];
|
||||
if (process.env.CODEGRAPH_KERNEL_PATH) candidates.push(process.env.CODEGRAPH_KERNEL_PATH);
|
||||
const packageRoot = path.resolve(__dirname, '..', '..', '..');
|
||||
candidates.push(path.join(packageRoot, 'kernel', 'codegraph-kernel.node'));
|
||||
candidates.push(
|
||||
path.join(
|
||||
packageRoot,
|
||||
'codegraph-kernel',
|
||||
'prebuilds',
|
||||
`${process.platform}-${process.arch}`,
|
||||
'codegraph-kernel.node'
|
||||
)
|
||||
);
|
||||
return candidates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify the binary speaks our wire contract: same ABI version and byte-equal
|
||||
* NodeKind/EdgeKind tables (kinds cross the boundary as indexes into these).
|
||||
*/
|
||||
function verifyContract(mod: KernelModule, from: string): boolean {
|
||||
const info = mod.contractInfo();
|
||||
if (info.abiVersion !== KERNEL_ABI_VERSION) {
|
||||
debug(`${from}: ABI ${info.abiVersion} != expected ${KERNEL_ABI_VERSION} — ignoring kernel`);
|
||||
return false;
|
||||
}
|
||||
const sameTable = (a: readonly string[], b: readonly string[]) =>
|
||||
a.length === b.length && a.every((v, i) => v === b[i]);
|
||||
if (!sameTable(info.nodeKinds, NODE_KINDS) || !sameTable(info.edgeKinds, EDGE_KINDS)) {
|
||||
debug(`${from}: NodeKind/EdgeKind tables differ from src/types.ts — ignoring kernel`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load (once per process) and return the kernel module, or null when
|
||||
* unavailable. The kill switch is NOT checked here — callers route through
|
||||
* `kernelAvailable()` / `tryKernelExtract()` which check it per call.
|
||||
*/
|
||||
export function getKernel(): KernelModule | null {
|
||||
if (cached !== undefined) return cached;
|
||||
cached = null;
|
||||
for (const candidate of candidatePaths()) {
|
||||
try {
|
||||
if (!fs.existsSync(candidate)) continue;
|
||||
// createRequire: works identically from CJS output and future ESM.
|
||||
const req = createRequire(__filename);
|
||||
const mod = req(candidate) as KernelModule;
|
||||
if (typeof mod.extractFile !== 'function' || typeof mod.contractInfo !== 'function') {
|
||||
debug(`${candidate}: missing expected exports — ignoring`);
|
||||
continue;
|
||||
}
|
||||
if (!verifyContract(mod, candidate)) continue;
|
||||
kernelLanguages = new Set(mod.contractInfo().languages);
|
||||
debug(`loaded ${candidate} (languages: ${[...kernelLanguages].join(', ')})`);
|
||||
cached = mod;
|
||||
break;
|
||||
} catch (err) {
|
||||
debug(`${candidate}: failed to load — ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
|
||||
/** True when the kill switch is off, a verified binary is loaded, and it supports `language`. */
|
||||
export function kernelSupports(language: string): boolean {
|
||||
if (process.env.CODEGRAPH_KERNEL === '0') return false;
|
||||
return getKernel() !== null && kernelLanguages.has(language);
|
||||
}
|
||||
|
||||
/** Test hook: forget the loaded module so a changed env is re-evaluated. */
|
||||
export function resetKernelForTests(): void {
|
||||
cached = undefined;
|
||||
kernelLanguages = new Set();
|
||||
}
|
||||
@@ -30,6 +30,7 @@ import { DfmExtractor } from './dfm-extractor';
|
||||
import { VueExtractor } from './vue-extractor';
|
||||
import { MyBatisExtractor } from './mybatis-extractor';
|
||||
import { CfmlExtractor } from './cfml-extractor';
|
||||
import { tryKernelExtract } from './kernel';
|
||||
import {
|
||||
getAllFrameworkResolvers,
|
||||
getApplicableFrameworks,
|
||||
@@ -6700,8 +6701,16 @@ export function extractFromSource(
|
||||
const extractor = new DfmExtractor(filePath, source);
|
||||
result = extractor.extract();
|
||||
} else {
|
||||
const extractor = new TreeSitterExtractor(filePath, source, detectedLanguage);
|
||||
result = extractor.extract();
|
||||
// Native-kernel route (docs/design/rust-kernel-migration-plan.md): gated
|
||||
// per language, null when not routed/available or on a kernel error —
|
||||
// the wasm TreeSitterExtractor below stays the fallback either way.
|
||||
const kernelResult = tryKernelExtract(filePath, source, detectedLanguage);
|
||||
if (kernelResult) {
|
||||
result = kernelResult;
|
||||
} else {
|
||||
const extractor = new TreeSitterExtractor(filePath, source, detectedLanguage);
|
||||
result = extractor.extract();
|
||||
}
|
||||
}
|
||||
|
||||
// Framework-specific extraction (routes, middleware, etc.)
|
||||
|
||||
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
BIN
Binary file not shown.
+25
-14
@@ -14,6 +14,10 @@
|
||||
* Defined as a runtime-iterable `as const` array so the same source
|
||||
* of truth backs both the TS type and any runtime validation
|
||||
* (e.g. the search query parser).
|
||||
*
|
||||
* The ARRAY ORDER is part of the native kernel's wire contract (kinds cross
|
||||
* the boundary as indexes — see src/extraction/kernel/layout.ts); append new
|
||||
* kinds, never reorder.
|
||||
*/
|
||||
export const NODE_KINDS = [
|
||||
'file',
|
||||
@@ -43,21 +47,28 @@ export const NODE_KINDS = [
|
||||
export type NodeKind = (typeof NODE_KINDS)[number];
|
||||
|
||||
/**
|
||||
* Types of edges (relationships) between nodes
|
||||
* Types of edges (relationships) between nodes.
|
||||
*
|
||||
* Runtime-iterable like NODE_KINDS. The ARRAY ORDER is part of the native
|
||||
* kernel's wire contract (kinds cross the boundary as indexes — see
|
||||
* src/extraction/kernel/layout.ts); append new kinds, never reorder.
|
||||
*/
|
||||
export type EdgeKind =
|
||||
| 'contains' // Parent contains child (file→class, class→method)
|
||||
| 'calls' // Function/method calls another
|
||||
| 'imports' // File imports from another
|
||||
| 'exports' // File exports a symbol
|
||||
| 'extends' // Class/interface extends another
|
||||
| 'implements' // Class implements interface
|
||||
| 'references' // Generic reference to another symbol
|
||||
| 'type_of' // Variable/parameter has type
|
||||
| 'returns' // Function returns type
|
||||
| 'instantiates' // Creates instance of class
|
||||
| 'overrides' // Method overrides parent method
|
||||
| 'decorates'; // Decorator applied to symbol
|
||||
export const EDGE_KINDS = [
|
||||
'contains', // Parent contains child (file→class, class→method)
|
||||
'calls', // Function/method calls another
|
||||
'imports', // File imports from another
|
||||
'exports', // File exports a symbol
|
||||
'extends', // Class/interface extends another
|
||||
'implements', // Class implements interface
|
||||
'references', // Generic reference to another symbol
|
||||
'type_of', // Variable/parameter has type
|
||||
'returns', // Function returns type
|
||||
'instantiates', // Creates instance of class
|
||||
'overrides', // Method overrides parent method
|
||||
'decorates', // Decorator applied to symbol
|
||||
] as const;
|
||||
|
||||
export type EdgeKind = (typeof EDGE_KINDS)[number];
|
||||
|
||||
/**
|
||||
* Supported programming languages. See NODE_KINDS for why this is a
|
||||
|
||||
Reference in New Issue
Block a user