feat(kernel): R1 scaffold — napi-rs extraction kernel, buffer contract, routing + fallback, grammar-parity CI
Phase 0 of the Rust extraction-kernel migration (docs/design/ rust-kernel-migration-plan.md, now checked in with §3a recording the shipped state): - codegraph-kernel/ napi-rs crate: extractFile(path, content, language) → five flat buffers (meta/nodes/edges/refs/arena), one JS boundary crossing per file. Node ids computed Rust-side, byte-identical to generateNodeId (pinned by test vector). Reserved per-node metrics slot for the Arc 3.2 code-metrics work. - Generic .scm-driven emitter (@def.<kind>/@name/@ref.<kind> captures, byte-range scope stack → ::-joined qualified names, contains edges, refs attributed to the innermost enclosing symbol). Seed TS/JS queries are smoke-level; R2 replaces them with the full port. - Routing seam in extractFromSource with per-file wasm fallback. DEFAULT_ROUTED is empty — no behavior change until a language passes its equivalence gate (R3). Dev opt-in: CODEGRAPH_KERNEL_LANGS. Kill switch: CODEGRAPH_KERNEL=0. Loader verifies ABI + kind tables before routing; EDGE_KINDS became a runtime array because kind order is now wire contract. - Grammar-source parity: vendored TS/TSX/JS wasm grammars built from the exact crate revisions (tree-sitter-typescript v0.23.2, tree-sitter-javascript v0.25.0, checked-in parser.c, ts-cli 0.25.10) — the tree-sitter-wasms builds were 2023-era, which the new kernel-grammar-parity test caught on day one. Production TS/JS parsing gets 2.5 years of grammar fixes; full suite green (2456 tests). - Build/release wiring: scripts/build-kernel.sh + npm run build:kernel; release.yml kernel prebuild matrix (continue-on-error — the kernel is optional everywhere, bundles fall back to the wasm path); bundles stage lib/kernel/codegraph-kernel.node; release job runs the kernel suites with CODEGRAPH_KERNEL_EXPECT=1. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
co-authored by
Claude Fable 5
Claude Fable 5
parent
4efc6c70e2
commit
c5eebe6beb
@@ -0,0 +1,3 @@
|
||||
target/
|
||||
prebuilds/
|
||||
*.node
|
||||
Generated
+542
@@ -0,0 +1,542 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.67"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "codegraph-kernel"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"napi",
|
||||
"napi-build",
|
||||
"napi-derive",
|
||||
"sha2",
|
||||
"streaming-iterator",
|
||||
"tree-sitter",
|
||||
"tree-sitter-javascript",
|
||||
"tree-sitter-typescript",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "convert_case"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49"
|
||||
dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctor"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a394189d59f9befacce833f337f7b1eca5e9a91221bcdd4d28e0114d96e597b3"
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-executor",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-macro",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.186"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "napi"
|
||||
version = "3.10.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6826e5ddc15589b2d68c8ad5321c18e85d40488e93e32962f362e572669bccf6"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"ctor",
|
||||
"futures",
|
||||
"napi-build",
|
||||
"napi-sys",
|
||||
"nohash-hasher",
|
||||
"rustc-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "napi-build"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1"
|
||||
|
||||
[[package]]
|
||||
name = "napi-derive"
|
||||
version = "3.5.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0fe526e81c105d3640516fcde83909dd1afe757c0d7a15af58830b5bc0fb9a1"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"ctor",
|
||||
"napi-derive-backend",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "napi-derive-backend"
|
||||
version = "5.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "514281397bcddd9ea9a876c7a21a57bff2374237a000ca9a64ea0211ec1993e2"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"semver",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "napi-sys"
|
||||
version = "3.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73e43cf2eb0bd1bf95a43c07c076ebd2da5d1e015a71c3d201faeffffcc0ecac"
|
||||
dependencies = [
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nohash-hasher"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.150"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter"
|
||||
version = "0.25.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"regex",
|
||||
"regex-syntax",
|
||||
"serde_json",
|
||||
"streaming-iterator",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-javascript"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-language"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-typescript"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
@@ -0,0 +1,32 @@
|
||||
[package]
|
||||
name = "codegraph-kernel"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "MIT"
|
||||
publish = false
|
||||
description = "Native extraction kernel for CodeGraph — tree-sitter parse+extract with one JS boundary crossing per file"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
napi = { version = "3", default-features = false, features = ["napi8"] }
|
||||
napi-derive = "3"
|
||||
tree-sitter = "0.25"
|
||||
streaming-iterator = "0.1"
|
||||
sha2 = "0.10"
|
||||
|
||||
# Grammars — MUST stay revision-matched with the wasm grammars the fallback
|
||||
# path loads (tree-sitter-wasms npm package / src/extraction/wasm/). The
|
||||
# kernel-grammar-parity test asserts node-kind-table equality at test time;
|
||||
# bump these together with the wasm side or that gate fails.
|
||||
tree-sitter-typescript = "0.23"
|
||||
tree-sitter-javascript = "0.25"
|
||||
|
||||
[build-dependencies]
|
||||
napi-build = "2"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
strip = "symbols"
|
||||
@@ -0,0 +1,3 @@
|
||||
fn main() {
|
||||
napi_build::setup();
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
; Seed query for the R1 scaffold (JavaScript / JSX grammar) — smoke-level
|
||||
; coverage only; R2 replaces this with the full port. See typescript.scm for
|
||||
; the capture convention.
|
||||
|
||||
(class_declaration name: (identifier) @name) @def.class
|
||||
(function_declaration name: (identifier) @name) @def.function
|
||||
(generator_function_declaration name: (identifier) @name) @def.function
|
||||
(method_definition name: (property_identifier) @name) @def.method
|
||||
|
||||
(call_expression function: (identifier) @ref.calls)
|
||||
(call_expression function: (member_expression property: (property_identifier) @ref.calls))
|
||||
(new_expression constructor: (identifier) @ref.instantiates)
|
||||
@@ -0,0 +1,21 @@
|
||||
; Seed query for the R1 scaffold — smoke-level coverage that proves the
|
||||
; buffer contract and emitter mechanics end to end. NOT extraction parity:
|
||||
; R2 replaces this with the full TypeScript/TSX port.
|
||||
;
|
||||
; Capture convention (see emitter.rs):
|
||||
; @def.<NodeKind> — the declaration node; pairs with @name in the pattern
|
||||
; @name — the declaration's name node
|
||||
; @ref.<EdgeKind> — a reference; the capture's own text is the name
|
||||
|
||||
(class_declaration name: (type_identifier) @name) @def.class
|
||||
(abstract_class_declaration name: (type_identifier) @name) @def.class
|
||||
(interface_declaration name: (type_identifier) @name) @def.interface
|
||||
(enum_declaration name: (identifier) @name) @def.enum
|
||||
(type_alias_declaration name: (type_identifier) @name) @def.type_alias
|
||||
(function_declaration name: (identifier) @name) @def.function
|
||||
(generator_function_declaration name: (identifier) @name) @def.function
|
||||
(method_definition name: (property_identifier) @name) @def.method
|
||||
|
||||
(call_expression function: (identifier) @ref.calls)
|
||||
(call_expression function: (member_expression property: (property_identifier) @ref.calls))
|
||||
(new_expression constructor: (identifier) @ref.instantiates)
|
||||
@@ -0,0 +1,387 @@
|
||||
//! Flat buffer contract — the ONE boundary crossing per file.
|
||||
//!
|
||||
//! The kernel returns five Buffers: meta, nodes, edges, refs, arena. All rows
|
||||
//! are fixed-width little-endian; every string is an (offset, len) pair into
|
||||
//! the UTF-8 arena. `OFFSET == NONE (0xFFFF_FFFF)` means "field absent".
|
||||
//!
|
||||
//! THIS FILE AND `src/extraction/kernel/layout.ts` MUST MATCH BYTE FOR BYTE.
|
||||
//! Any layout change bumps `KERNEL_ABI_VERSION` — the TS loader refuses a
|
||||
//! version it doesn't know and falls back to the wasm path.
|
||||
//!
|
||||
//! Layout (v1):
|
||||
//!
|
||||
//! meta (36 bytes):
|
||||
//! 0 u8 KERNEL_ABI_VERSION
|
||||
//! 1 [3] pad
|
||||
//! 4 u32 node count
|
||||
//! 8 u32 edge count
|
||||
//! 12 u32 ref count
|
||||
//! 16 u32 arena byte length
|
||||
//! 20 u32 errors-JSON arena offset (NONE = no errors)
|
||||
//! 24 u32 errors-JSON byte length
|
||||
//! 28 f64 kernel-side wall duration (ms) — introspection only; the TS
|
||||
//! wrapper measures the ExtractionResult.durationMs it reports
|
||||
//!
|
||||
//! node row (96 bytes):
|
||||
//! 0 u8 NodeKind index (NODE_KINDS order)
|
||||
//! 1 u8 visibility (0 absent, 1 public, 2 private, 3 protected, 4 internal)
|
||||
//! 2 u16 bool flags — bit pairs (present, value):
|
||||
//! 0/1 isExported, 2/3 isAsync, 4/5 isStatic, 6/7 isAbstract
|
||||
//! 4 u32 startLine (1-based)
|
||||
//! 8 u32 endLine
|
||||
//! 12 u32 startColumn (0-based)
|
||||
//! 16 u32 endColumn
|
||||
//! 20 str name
|
||||
//! 28 str qualifiedName
|
||||
//! 36 str id (kernel-computed: "kind:hash32", or "file:<path>" for the file node)
|
||||
//! 44 str docstring
|
||||
//! 52 str signature
|
||||
//! 60 str decorators (NUL-joined list)
|
||||
//! 68 str typeParameters (NUL-joined list)
|
||||
//! 76 str returnType
|
||||
//! 84 str extraJson (escape hatch: JSON of any extra Node props)
|
||||
//! 92 u32 metrics slot (reserved for Arc 3.2 per-node code metrics; 0)
|
||||
//!
|
||||
//! edge row (44 bytes):
|
||||
//! 0 u32 source node row index (NONE → use sourceIdStr)
|
||||
//! 4 u32 target node row index (NONE → use targetIdStr)
|
||||
//! 8 u8 EdgeKind index (EDGE_KINDS order)
|
||||
//! 9 u8 provenance (0 absent, 1 tree-sitter, 2 scip, 3 heuristic)
|
||||
//! 10 u16 pad
|
||||
//! 12 u32 line (NONE absent)
|
||||
//! 16 u32 column (NONE absent)
|
||||
//! 20 str metadataJson
|
||||
//! 28 str sourceIdStr
|
||||
//! 36 str targetIdStr
|
||||
//!
|
||||
//! ref row (40 bytes):
|
||||
//! 0 u32 fromNode row index (NONE → use fromNodeIdStr)
|
||||
//! 4 u8 ReferenceKind (EDGE_KINDS index, or 200 = function_ref)
|
||||
//! 5 [3] pad
|
||||
//! 8 u32 line (1-based)
|
||||
//! 12 u32 column (0-based)
|
||||
//! 16 str referenceName
|
||||
//! 24 str candidates (NUL-joined list)
|
||||
//! 32 str fromNodeIdStr
|
||||
|
||||
pub const KERNEL_ABI_VERSION: u8 = 1;
|
||||
pub const NONE: u32 = 0xFFFF_FFFF;
|
||||
|
||||
pub const META_SIZE: usize = 36;
|
||||
pub const NODE_ROW_SIZE: usize = 96;
|
||||
pub const EDGE_ROW_SIZE: usize = 44;
|
||||
pub const REF_ROW_SIZE: usize = 40;
|
||||
|
||||
/// Mirror of NODE_KINDS in src/types.ts — order is the wire contract.
|
||||
pub const NODE_KINDS: [&str; 22] = [
|
||||
"file",
|
||||
"module",
|
||||
"class",
|
||||
"struct",
|
||||
"interface",
|
||||
"trait",
|
||||
"protocol",
|
||||
"function",
|
||||
"method",
|
||||
"property",
|
||||
"field",
|
||||
"variable",
|
||||
"constant",
|
||||
"enum",
|
||||
"enum_member",
|
||||
"type_alias",
|
||||
"namespace",
|
||||
"parameter",
|
||||
"import",
|
||||
"export",
|
||||
"route",
|
||||
"component",
|
||||
];
|
||||
|
||||
/// Mirror of EDGE_KINDS in src/types.ts — order is the wire contract.
|
||||
pub const EDGE_KINDS: [&str; 12] = [
|
||||
"contains",
|
||||
"calls",
|
||||
"imports",
|
||||
"exports",
|
||||
"extends",
|
||||
"implements",
|
||||
"references",
|
||||
"type_of",
|
||||
"returns",
|
||||
"instantiates",
|
||||
"overrides",
|
||||
"decorates",
|
||||
];
|
||||
|
||||
/// ReferenceKind code for the internal-only `function_ref` (#756).
|
||||
pub const FUNCTION_REF_CODE: u8 = 200;
|
||||
|
||||
pub fn node_kind_index(kind: &str) -> Option<u8> {
|
||||
NODE_KINDS.iter().position(|k| *k == kind).map(|i| i as u8)
|
||||
}
|
||||
|
||||
pub fn edge_kind_index(kind: &str) -> Option<u8> {
|
||||
EDGE_KINDS.iter().position(|k| *k == kind).map(|i| i as u8)
|
||||
}
|
||||
|
||||
/// (offset, len) arena reference. `NONE_STR` encodes an absent field.
|
||||
pub type StrRef = (u32, u32);
|
||||
pub const NONE_STR: StrRef = (NONE, 0);
|
||||
|
||||
/// UTF-8 string arena. Strings are appended verbatim; no dedup (per-file
|
||||
/// buffers are transient and small — intern later if profiling says so).
|
||||
#[derive(Default)]
|
||||
pub struct Arena {
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Arena {
|
||||
pub fn put(&mut self, s: &str) -> StrRef {
|
||||
let off = self.buf.len() as u32;
|
||||
self.buf.extend_from_slice(s.as_bytes());
|
||||
(off, s.len() as u32)
|
||||
}
|
||||
|
||||
/// Not used by the seed emitter yet — R2 (docstring/signature/etc.). Kept
|
||||
/// so the arena API is complete alongside the layout it feeds.
|
||||
#[allow(dead_code)]
|
||||
pub fn put_opt(&mut self, s: Option<&str>) -> StrRef {
|
||||
match s {
|
||||
Some(s) => self.put(s),
|
||||
None => NONE_STR,
|
||||
}
|
||||
}
|
||||
|
||||
/// NUL-joined list; absent when the list is empty. (R2 surface: decorators,
|
||||
/// typeParameters, candidates.)
|
||||
#[allow(dead_code)]
|
||||
pub fn put_list(&mut self, items: &[String]) -> StrRef {
|
||||
if items.is_empty() {
|
||||
return NONE_STR;
|
||||
}
|
||||
let joined = items.join("\0");
|
||||
self.put(&joined)
|
||||
}
|
||||
|
||||
pub fn len(&self) -> u32 {
|
||||
self.buf.len() as u32
|
||||
}
|
||||
|
||||
pub fn into_vec(self) -> Vec<u8> {
|
||||
self.buf
|
||||
}
|
||||
}
|
||||
|
||||
/// Tri-state booleans packed as (present, value) bit pairs.
|
||||
#[derive(Default, Clone, Copy)]
|
||||
pub struct BoolFlags(pub u16);
|
||||
|
||||
impl BoolFlags {
|
||||
pub fn set(&mut self, pair: u16, value: bool) {
|
||||
self.0 |= 1 << (pair * 2);
|
||||
if value {
|
||||
self.0 |= 1 << (pair * 2 + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub const FLAG_IS_EXPORTED: u16 = 0;
|
||||
#[allow(dead_code)] // R2 surface — part of the v1 wire contract
|
||||
pub const FLAG_IS_ASYNC: u16 = 1;
|
||||
#[allow(dead_code)] // R2 surface — part of the v1 wire contract
|
||||
pub const FLAG_IS_STATIC: u16 = 2;
|
||||
#[allow(dead_code)] // R2 surface — part of the v1 wire contract
|
||||
pub const FLAG_IS_ABSTRACT: u16 = 3;
|
||||
|
||||
pub struct NodeRow {
|
||||
pub kind: u8,
|
||||
pub visibility: u8,
|
||||
pub flags: BoolFlags,
|
||||
pub start_line: u32,
|
||||
pub end_line: u32,
|
||||
pub start_column: u32,
|
||||
pub end_column: u32,
|
||||
pub name: StrRef,
|
||||
pub qualified_name: StrRef,
|
||||
pub id: StrRef,
|
||||
pub docstring: StrRef,
|
||||
pub signature: StrRef,
|
||||
pub decorators: StrRef,
|
||||
pub type_parameters: StrRef,
|
||||
pub return_type: StrRef,
|
||||
pub extra_json: StrRef,
|
||||
}
|
||||
|
||||
pub struct EdgeRow {
|
||||
pub source_idx: u32,
|
||||
pub target_idx: u32,
|
||||
pub kind: u8,
|
||||
pub provenance: u8,
|
||||
pub line: u32,
|
||||
pub column: u32,
|
||||
pub metadata_json: StrRef,
|
||||
pub source_id_str: StrRef,
|
||||
pub target_id_str: StrRef,
|
||||
}
|
||||
|
||||
pub struct RefRow {
|
||||
pub from_idx: u32,
|
||||
pub kind: u8,
|
||||
pub line: u32,
|
||||
pub column: u32,
|
||||
pub reference_name: StrRef,
|
||||
pub candidates: StrRef,
|
||||
pub from_id_str: StrRef,
|
||||
}
|
||||
|
||||
fn push_str_ref(buf: &mut Vec<u8>, r: StrRef) {
|
||||
buf.extend_from_slice(&r.0.to_le_bytes());
|
||||
buf.extend_from_slice(&r.1.to_le_bytes());
|
||||
}
|
||||
|
||||
pub struct Tables {
|
||||
pub nodes: Vec<u8>,
|
||||
pub edges: Vec<u8>,
|
||||
pub refs: Vec<u8>,
|
||||
pub node_count: u32,
|
||||
pub edge_count: u32,
|
||||
pub ref_count: u32,
|
||||
}
|
||||
|
||||
impl Default for Tables {
|
||||
fn default() -> Self {
|
||||
Tables {
|
||||
nodes: Vec::with_capacity(NODE_ROW_SIZE * 64),
|
||||
edges: Vec::with_capacity(EDGE_ROW_SIZE * 64),
|
||||
refs: Vec::with_capacity(REF_ROW_SIZE * 64),
|
||||
node_count: 0,
|
||||
edge_count: 0,
|
||||
ref_count: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Tables {
|
||||
pub fn push_node(&mut self, r: &NodeRow) -> u32 {
|
||||
let buf = &mut self.nodes;
|
||||
buf.push(r.kind);
|
||||
buf.push(r.visibility);
|
||||
buf.extend_from_slice(&r.flags.0.to_le_bytes());
|
||||
buf.extend_from_slice(&r.start_line.to_le_bytes());
|
||||
buf.extend_from_slice(&r.end_line.to_le_bytes());
|
||||
buf.extend_from_slice(&r.start_column.to_le_bytes());
|
||||
buf.extend_from_slice(&r.end_column.to_le_bytes());
|
||||
push_str_ref(buf, r.name);
|
||||
push_str_ref(buf, r.qualified_name);
|
||||
push_str_ref(buf, r.id);
|
||||
push_str_ref(buf, r.docstring);
|
||||
push_str_ref(buf, r.signature);
|
||||
push_str_ref(buf, r.decorators);
|
||||
push_str_ref(buf, r.type_parameters);
|
||||
push_str_ref(buf, r.return_type);
|
||||
push_str_ref(buf, r.extra_json);
|
||||
buf.extend_from_slice(&0u32.to_le_bytes()); // metrics slot (Arc 3.2)
|
||||
let idx = self.node_count;
|
||||
self.node_count += 1;
|
||||
idx
|
||||
}
|
||||
|
||||
pub fn push_edge(&mut self, r: &EdgeRow) {
|
||||
let buf = &mut self.edges;
|
||||
buf.extend_from_slice(&r.source_idx.to_le_bytes());
|
||||
buf.extend_from_slice(&r.target_idx.to_le_bytes());
|
||||
buf.push(r.kind);
|
||||
buf.push(r.provenance);
|
||||
buf.extend_from_slice(&0u16.to_le_bytes()); // pad
|
||||
buf.extend_from_slice(&r.line.to_le_bytes());
|
||||
buf.extend_from_slice(&r.column.to_le_bytes());
|
||||
push_str_ref(buf, r.metadata_json);
|
||||
push_str_ref(buf, r.source_id_str);
|
||||
push_str_ref(buf, r.target_id_str);
|
||||
self.edge_count += 1;
|
||||
}
|
||||
|
||||
pub fn push_ref(&mut self, r: &RefRow) {
|
||||
let buf = &mut self.refs;
|
||||
buf.extend_from_slice(&r.from_idx.to_le_bytes());
|
||||
buf.push(r.kind);
|
||||
buf.extend_from_slice(&[0u8; 3]); // pad
|
||||
buf.extend_from_slice(&r.line.to_le_bytes());
|
||||
buf.extend_from_slice(&r.column.to_le_bytes());
|
||||
push_str_ref(buf, r.reference_name);
|
||||
push_str_ref(buf, r.candidates);
|
||||
push_str_ref(buf, r.from_id_str);
|
||||
self.ref_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_meta(t: &Tables, arena_len: u32, errors_json: StrRef, duration_ms: f64) -> Vec<u8> {
|
||||
let mut m = Vec::with_capacity(META_SIZE);
|
||||
m.push(KERNEL_ABI_VERSION);
|
||||
m.extend_from_slice(&[0u8; 3]);
|
||||
m.extend_from_slice(&t.node_count.to_le_bytes());
|
||||
m.extend_from_slice(&t.edge_count.to_le_bytes());
|
||||
m.extend_from_slice(&t.ref_count.to_le_bytes());
|
||||
m.extend_from_slice(&arena_len.to_le_bytes());
|
||||
m.extend_from_slice(&errors_json.0.to_le_bytes());
|
||||
m.extend_from_slice(&errors_json.1.to_le_bytes());
|
||||
m.extend_from_slice(&duration_ms.to_le_bytes());
|
||||
debug_assert_eq!(m.len(), META_SIZE);
|
||||
m
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn row_sizes_match_constants() {
|
||||
let mut t = Tables::default();
|
||||
let mut a = Arena::default();
|
||||
let name = a.put("x");
|
||||
t.push_node(&NodeRow {
|
||||
kind: 0,
|
||||
visibility: 0,
|
||||
flags: BoolFlags::default(),
|
||||
start_line: 1,
|
||||
end_line: 1,
|
||||
start_column: 0,
|
||||
end_column: 0,
|
||||
name,
|
||||
qualified_name: name,
|
||||
id: name,
|
||||
docstring: NONE_STR,
|
||||
signature: NONE_STR,
|
||||
decorators: NONE_STR,
|
||||
type_parameters: NONE_STR,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
assert_eq!(t.nodes.len(), NODE_ROW_SIZE);
|
||||
t.push_edge(&EdgeRow {
|
||||
source_idx: 0,
|
||||
target_idx: 0,
|
||||
kind: 0,
|
||||
provenance: 0,
|
||||
line: NONE,
|
||||
column: NONE,
|
||||
metadata_json: NONE_STR,
|
||||
source_id_str: NONE_STR,
|
||||
target_id_str: NONE_STR,
|
||||
});
|
||||
assert_eq!(t.edges.len(), EDGE_ROW_SIZE);
|
||||
t.push_ref(&RefRow {
|
||||
from_idx: 0,
|
||||
kind: 1,
|
||||
line: 1,
|
||||
column: 0,
|
||||
reference_name: name,
|
||||
candidates: NONE_STR,
|
||||
from_id_str: NONE_STR,
|
||||
});
|
||||
assert_eq!(t.refs.len(), REF_ROW_SIZE);
|
||||
let meta = build_meta(&t, a.len(), NONE_STR, 0.0);
|
||||
assert_eq!(meta.len(), META_SIZE);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
//! Generic query-driven emitter: parse the file, run the language's `.scm`
|
||||
//! query, and emit flat rows. The whole tree walk happens native-side; the
|
||||
//! only JS boundary crossing is the returned buffers.
|
||||
//!
|
||||
//! Mechanics mirrored from `TreeSitterExtractor` (src/extraction/tree-sitter.ts):
|
||||
//! - node row 0 is the file node (`file:<path>`, endLine = newline count + 1,
|
||||
//! isExported present+false — byte-parity with the TS file node);
|
||||
//! - definitions form a scope stack by byte-range nesting; qualifiedName is
|
||||
//! the stack's names joined with `::` (file excluded);
|
||||
//! - every definition gets a `contains` edge from its parent scope (the
|
||||
//! file node when top-level);
|
||||
//! - references attach to the innermost enclosing definition, falling back
|
||||
//! to the file node — same as the TS extractor's nodeStack semantics;
|
||||
//! - definitions with empty names are skipped (issue #42 semantics).
|
||||
|
||||
use crate::buffers::{
|
||||
build_meta, edge_kind_index, node_kind_index, Arena, BoolFlags, EdgeRow, NodeRow, RefRow,
|
||||
Tables, FLAG_IS_EXPORTED, FUNCTION_REF_CODE, NODE_KINDS, NONE, NONE_STR,
|
||||
};
|
||||
use crate::ids;
|
||||
use crate::langs::LangSpec;
|
||||
use streaming_iterator::StreamingIterator;
|
||||
use tree_sitter::{Node, Parser, QueryCursor};
|
||||
|
||||
pub struct EmitOut {
|
||||
pub meta: Vec<u8>,
|
||||
pub nodes: Vec<u8>,
|
||||
pub edges: Vec<u8>,
|
||||
pub refs: Vec<u8>,
|
||||
pub arena: Vec<u8>,
|
||||
}
|
||||
|
||||
/// What a query capture name means. Resolved once per query.
|
||||
#[derive(Clone, Copy)]
|
||||
enum Role {
|
||||
/// `@def.<NodeKind>` — value is the NODE_KINDS index.
|
||||
Def(u8),
|
||||
/// `@name` — the paired definition's name node.
|
||||
Name,
|
||||
/// `@ref.<EdgeKind>` / `@ref.function_ref` — value is the wire code.
|
||||
Ref(u8),
|
||||
/// Helper captures (`@_anchor` etc.) — ignored.
|
||||
Ignore,
|
||||
}
|
||||
|
||||
fn resolve_roles(capture_names: &[&str], lang: &str) -> Result<Vec<Role>, String> {
|
||||
capture_names
|
||||
.iter()
|
||||
.map(|name| {
|
||||
if let Some(kind) = name.strip_prefix("def.") {
|
||||
let idx = node_kind_index(kind)
|
||||
.ok_or_else(|| format!("{lang}: unknown NodeKind in capture @{name}"))?;
|
||||
Ok(Role::Def(idx))
|
||||
} else if let Some(kind) = name.strip_prefix("ref.") {
|
||||
if kind == "function_ref" {
|
||||
return Ok(Role::Ref(FUNCTION_REF_CODE));
|
||||
}
|
||||
let idx = edge_kind_index(kind)
|
||||
.ok_or_else(|| format!("{lang}: unknown EdgeKind in capture @{name}"))?;
|
||||
Ok(Role::Ref(idx))
|
||||
} else if *name == "name" {
|
||||
Ok(Role::Name)
|
||||
} else {
|
||||
Ok(Role::Ignore)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
struct Def {
|
||||
kind: u8,
|
||||
name_start: usize,
|
||||
name_end: usize,
|
||||
start_byte: usize,
|
||||
end_byte: usize,
|
||||
start_line: u32,
|
||||
end_line: u32,
|
||||
start_column: u32,
|
||||
end_column: u32,
|
||||
/// Node-table row index, assigned during the scope sweep.
|
||||
row: u32,
|
||||
}
|
||||
|
||||
struct RefCap {
|
||||
kind: u8,
|
||||
name_start: usize,
|
||||
name_end: usize,
|
||||
start_byte: usize,
|
||||
line: u32,
|
||||
column: u32,
|
||||
}
|
||||
|
||||
pub fn extract(file_path: &str, source: &str, spec: &LangSpec) -> Result<EmitOut, String> {
|
||||
let t0 = std::time::Instant::now();
|
||||
|
||||
let mut parser = Parser::new();
|
||||
parser
|
||||
.set_language(spec.language())
|
||||
.map_err(|e| format!("set_language({}) failed: {e}", spec.name))?;
|
||||
let tree = parser
|
||||
.parse(source, None)
|
||||
.ok_or_else(|| "parser returned null tree".to_string())?;
|
||||
let root = tree.root_node();
|
||||
|
||||
let query = spec.query()?;
|
||||
let roles = resolve_roles(&query.capture_names(), spec.name)?;
|
||||
|
||||
// ---- Collect definition + reference captures from the query. ----
|
||||
let mut defs: Vec<Def> = Vec::new();
|
||||
let mut refs: Vec<RefCap> = Vec::new();
|
||||
// A node can match several patterns (e.g. nested alternations); first
|
||||
// pattern wins, mirroring the TS walk's one-node-one-symbol behaviour.
|
||||
let mut seen_defs = std::collections::HashSet::<usize>::new();
|
||||
|
||||
let mut cursor = QueryCursor::new();
|
||||
let mut matches = cursor.matches(query, root, source.as_bytes());
|
||||
while let Some(m) = matches.next() {
|
||||
let mut def_node: Option<(Node, u8)> = None;
|
||||
let mut name_node: Option<Node> = None;
|
||||
for cap in m.captures {
|
||||
match roles[cap.index as usize] {
|
||||
Role::Def(kind) => def_node = Some((cap.node, kind)),
|
||||
Role::Name => name_node = Some(cap.node),
|
||||
Role::Ref(kind) => {
|
||||
let p = cap.node.start_position();
|
||||
refs.push(RefCap {
|
||||
kind,
|
||||
name_start: cap.node.start_byte(),
|
||||
name_end: cap.node.end_byte(),
|
||||
start_byte: cap.node.start_byte(),
|
||||
line: p.row as u32 + 1,
|
||||
column: p.column as u32,
|
||||
});
|
||||
}
|
||||
Role::Ignore => {}
|
||||
}
|
||||
}
|
||||
if let (Some((node, kind)), Some(name)) = (def_node, name_node) {
|
||||
// Empty names are not meaningful symbols (issue #42).
|
||||
if name.end_byte() > name.start_byte() && seen_defs.insert(node.id()) {
|
||||
let sp = node.start_position();
|
||||
let ep = node.end_position();
|
||||
defs.push(Def {
|
||||
kind,
|
||||
name_start: name.start_byte(),
|
||||
name_end: name.end_byte(),
|
||||
start_byte: node.start_byte(),
|
||||
end_byte: node.end_byte(),
|
||||
start_line: sp.row as u32 + 1,
|
||||
end_line: ep.row as u32 + 1,
|
||||
start_column: sp.column as u32,
|
||||
end_column: ep.column as u32,
|
||||
row: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deterministic pre-order regardless of query-match ordering.
|
||||
defs.sort_by(|a, b| {
|
||||
a.start_byte
|
||||
.cmp(&b.start_byte)
|
||||
.then(b.end_byte.cmp(&a.end_byte))
|
||||
});
|
||||
refs.sort_by_key(|r| r.start_byte);
|
||||
|
||||
// ---- Emit rows: file node first, then the scope-stack sweep. ----
|
||||
let mut arena = Arena::default();
|
||||
let mut tables = Tables::default();
|
||||
|
||||
let line_count = source.bytes().filter(|b| *b == b'\n').count() as u32 + 1;
|
||||
let base_name = file_path.rsplit(['/', '\\']).next().unwrap_or(file_path);
|
||||
let mut file_flags = BoolFlags::default();
|
||||
file_flags.set(FLAG_IS_EXPORTED, false);
|
||||
let file_id = arena.put(&ids::file_node_id(file_path));
|
||||
let file_name = arena.put(base_name);
|
||||
let file_qn = arena.put(file_path);
|
||||
tables.push_node(&NodeRow {
|
||||
kind: node_kind_index("file").unwrap(),
|
||||
visibility: 0,
|
||||
flags: file_flags,
|
||||
start_line: 1,
|
||||
end_line: line_count,
|
||||
start_column: 0,
|
||||
end_column: 0,
|
||||
name: file_name,
|
||||
qualified_name: file_qn,
|
||||
id: file_id,
|
||||
docstring: NONE_STR,
|
||||
signature: NONE_STR,
|
||||
decorators: NONE_STR,
|
||||
type_parameters: NONE_STR,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
|
||||
// Merged sweep over definitions and references in byte order, maintaining
|
||||
// the scope stack (indices into `defs`).
|
||||
let mut stack: Vec<usize> = Vec::new();
|
||||
let mut ref_i = 0usize;
|
||||
|
||||
fn pop_to(stack: &mut Vec<usize>, defs: &[Def], byte: usize) {
|
||||
while let Some(&top) = stack.last() {
|
||||
if defs[top].end_byte <= byte {
|
||||
stack.pop();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let emit_ref = |r: &RefCap, stack: &[usize], defs: &[Def], arena: &mut Arena, tables: &mut Tables| {
|
||||
let from_idx = stack.last().map(|&i| defs[i].row).unwrap_or(0);
|
||||
let name = arena.put(&source[r.name_start..r.name_end]);
|
||||
tables.push_ref(&RefRow {
|
||||
from_idx,
|
||||
kind: r.kind,
|
||||
line: r.line,
|
||||
column: r.column,
|
||||
reference_name: name,
|
||||
candidates: NONE_STR,
|
||||
from_id_str: NONE_STR,
|
||||
});
|
||||
};
|
||||
|
||||
for i in 0..defs.len() {
|
||||
let def_start = defs[i].start_byte;
|
||||
while ref_i < refs.len() && refs[ref_i].start_byte < def_start {
|
||||
pop_to(&mut stack, &defs, refs[ref_i].start_byte);
|
||||
emit_ref(&refs[ref_i], &stack, &defs, &mut arena, &mut tables);
|
||||
ref_i += 1;
|
||||
}
|
||||
pop_to(&mut stack, &defs, def_start);
|
||||
|
||||
let name = &source[defs[i].name_start..defs[i].name_end];
|
||||
let kind_str = NODE_KINDS[defs[i].kind as usize];
|
||||
// qualifiedName = enclosing definition names + own name, `::`-joined
|
||||
// (buildQualifiedName semantics; file node excluded).
|
||||
let mut qn = String::new();
|
||||
for &s in stack.iter() {
|
||||
qn.push_str(&source[defs[s].name_start..defs[s].name_end]);
|
||||
qn.push_str("::");
|
||||
}
|
||||
qn.push_str(name);
|
||||
|
||||
let id = ids::node_id(file_path, kind_str, name, defs[i].start_line);
|
||||
let id_ref = arena.put(&id);
|
||||
let name_ref = arena.put(name);
|
||||
let qn_ref = arena.put(&qn);
|
||||
let row = tables.push_node(&NodeRow {
|
||||
kind: defs[i].kind,
|
||||
visibility: 0,
|
||||
flags: BoolFlags::default(),
|
||||
start_line: defs[i].start_line,
|
||||
end_line: defs[i].end_line,
|
||||
start_column: defs[i].start_column,
|
||||
end_column: defs[i].end_column,
|
||||
name: name_ref,
|
||||
qualified_name: qn_ref,
|
||||
id: id_ref,
|
||||
docstring: NONE_STR,
|
||||
signature: NONE_STR,
|
||||
decorators: NONE_STR,
|
||||
type_parameters: NONE_STR,
|
||||
return_type: NONE_STR,
|
||||
extra_json: NONE_STR,
|
||||
});
|
||||
defs[i].row = row;
|
||||
|
||||
let parent_row = stack.last().map(|&s| defs[s].row).unwrap_or(0);
|
||||
tables.push_edge(&EdgeRow {
|
||||
source_idx: parent_row,
|
||||
target_idx: row,
|
||||
kind: edge_kind_index("contains").unwrap(),
|
||||
provenance: 0,
|
||||
line: NONE,
|
||||
column: NONE,
|
||||
metadata_json: NONE_STR,
|
||||
source_id_str: NONE_STR,
|
||||
target_id_str: NONE_STR,
|
||||
});
|
||||
|
||||
stack.push(i);
|
||||
}
|
||||
while ref_i < refs.len() {
|
||||
pop_to(&mut stack, &defs, refs[ref_i].start_byte);
|
||||
emit_ref(&refs[ref_i], &stack, &defs, &mut arena, &mut tables);
|
||||
ref_i += 1;
|
||||
}
|
||||
|
||||
let duration_ms = t0.elapsed().as_secs_f64() * 1000.0;
|
||||
let meta = build_meta(&tables, arena.len(), NONE_STR, duration_ms);
|
||||
Ok(EmitOut {
|
||||
meta,
|
||||
nodes: tables.nodes,
|
||||
edges: tables.edges,
|
||||
refs: tables.refs,
|
||||
arena: arena.into_vec(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
//! Node-ID generation — MUST produce byte-identical output to
|
||||
//! `generateNodeId` in `src/extraction/tree-sitter-helpers.ts`:
|
||||
//!
|
||||
//! `${kind}:${sha256(`${filePath}:${kind}:${name}:${line}`).hex[0..32]}`
|
||||
//!
|
||||
//! and the file-node special case in `TreeSitterExtractor.extract()`:
|
||||
//!
|
||||
//! `file:${filePath}`
|
||||
//!
|
||||
//! Node identity is how the wasm path and the kernel path agree on the same
|
||||
//! graph — a drift here breaks every edge. Pinned by the node-id parity test
|
||||
//! in `__tests__/kernel-scaffold.test.ts`.
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub fn node_id(file_path: &str, kind: &str, name: &str, line: u32) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(file_path.as_bytes());
|
||||
hasher.update(b":");
|
||||
hasher.update(kind.as_bytes());
|
||||
hasher.update(b":");
|
||||
hasher.update(name.as_bytes());
|
||||
hasher.update(b":");
|
||||
hasher.update(line.to_string().as_bytes());
|
||||
let digest = hasher.finalize();
|
||||
// 32 hex chars = first 16 bytes.
|
||||
let mut hex = String::with_capacity(kind.len() + 1 + 32);
|
||||
hex.push_str(kind);
|
||||
hex.push(':');
|
||||
for b in &digest[..16] {
|
||||
hex.push_str(&format!("{b:02x}"));
|
||||
}
|
||||
hex
|
||||
}
|
||||
|
||||
pub fn file_node_id(file_path: &str) -> String {
|
||||
format!("file:{file_path}")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn matches_known_ts_output() {
|
||||
// Pinned vector: node -e "crypto.createHash('sha256')
|
||||
// .update('src/a.ts:function:foo:3').digest('hex').substring(0,32)"
|
||||
assert_eq!(
|
||||
node_id("src/a.ts", "function", "foo", 3),
|
||||
"function:bfb15544fed707794274a5c61006ea7b"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
//! Per-language specs: grammar + `.scm` query + (later) per-language config.
|
||||
//!
|
||||
//! Tier-1 languages are meant to be *mostly* a query file plus a small config
|
||||
//! here; logic queries can't express stays TS-side as a per-language `post()`
|
||||
//! hook over the returned buffers (see `src/extraction/kernel/route.ts`).
|
||||
//!
|
||||
//! Language strings are codegraph `Language` values (src/types.ts), not
|
||||
//! grammar names — `tsx` and `jsx` are separate entries that reuse another
|
||||
//! entry's grammar exactly like `WASM_GRAMMAR_FILES` does on the wasm path.
|
||||
|
||||
use std::sync::OnceLock;
|
||||
use tree_sitter::{Language, Query};
|
||||
|
||||
pub struct LangSpec {
|
||||
/// codegraph Language string (src/types.ts).
|
||||
pub name: &'static str,
|
||||
get_language: fn() -> Language,
|
||||
query_src: &'static str,
|
||||
language: OnceLock<Language>,
|
||||
query: OnceLock<Result<Query, String>>,
|
||||
}
|
||||
|
||||
impl LangSpec {
|
||||
const fn new(name: &'static str, get_language: fn() -> Language, query_src: &'static str) -> Self {
|
||||
LangSpec {
|
||||
name,
|
||||
get_language,
|
||||
query_src,
|
||||
language: OnceLock::new(),
|
||||
query: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn language(&self) -> &Language {
|
||||
self.language.get_or_init(self.get_language)
|
||||
}
|
||||
|
||||
pub fn query(&self) -> Result<&Query, String> {
|
||||
self.query
|
||||
.get_or_init(|| {
|
||||
Query::new(self.language(), self.query_src)
|
||||
.map_err(|e| format!("query compile failed for {}: {e}", self.name))
|
||||
})
|
||||
.as_ref()
|
||||
.map_err(|e| e.clone())
|
||||
}
|
||||
}
|
||||
|
||||
fn ts_language() -> Language {
|
||||
tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()
|
||||
}
|
||||
|
||||
fn tsx_language() -> Language {
|
||||
tree_sitter_typescript::LANGUAGE_TSX.into()
|
||||
}
|
||||
|
||||
fn js_language() -> Language {
|
||||
tree_sitter_javascript::LANGUAGE.into()
|
||||
}
|
||||
|
||||
static TYPESCRIPT: LangSpec = LangSpec::new(
|
||||
"typescript",
|
||||
ts_language,
|
||||
include_str!("../queries/typescript.scm"),
|
||||
);
|
||||
static TSX: LangSpec = LangSpec::new("tsx", tsx_language, include_str!("../queries/typescript.scm"));
|
||||
static JAVASCRIPT: LangSpec = LangSpec::new(
|
||||
"javascript",
|
||||
js_language,
|
||||
include_str!("../queries/javascript.scm"),
|
||||
);
|
||||
static JSX: LangSpec = LangSpec::new("jsx", js_language, include_str!("../queries/javascript.scm"));
|
||||
|
||||
pub static ALL: [&LangSpec; 4] = [&TYPESCRIPT, &TSX, &JAVASCRIPT, &JSX];
|
||||
|
||||
pub fn spec_for(language: &str) -> Option<&'static LangSpec> {
|
||||
ALL.iter().find(|s| s.name == language).copied()
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
//! codegraph-kernel — native extraction kernel (napi-rs).
|
||||
//!
|
||||
//! Replaces ONLY the parse+extract walk inside the parse workers, behind the
|
||||
//! existing `ExtractionResult` contract. Input `(filePath, content, language)`
|
||||
//! per file; output flat typed buffers — one boundary crossing per file.
|
||||
//! Everything downstream (resolution, synthesis, frameworks, MCP) is
|
||||
//! untouched and consumes the decoded result exactly as before.
|
||||
//!
|
||||
//! Calls are synchronous by design: the existing `ParseWorkerPool` workers
|
||||
//! already parallelize per-file, so each worker thread drives its own kernel
|
||||
//! call (do NOT rebuild the pool on the Rust side — see the migration plan §3).
|
||||
|
||||
#![deny(clippy::all)]
|
||||
|
||||
mod buffers;
|
||||
mod emitter;
|
||||
mod ids;
|
||||
mod langs;
|
||||
|
||||
use napi::bindgen_prelude::*;
|
||||
use napi_derive::napi;
|
||||
|
||||
/// The five flat tables for one file. See buffers.rs for the byte layout;
|
||||
/// `src/extraction/kernel/layout.ts` is the TS mirror.
|
||||
#[napi(object)]
|
||||
pub struct ExtractBuffers {
|
||||
pub meta: Buffer,
|
||||
pub nodes: Buffer,
|
||||
pub edges: Buffer,
|
||||
pub refs: Buffer,
|
||||
pub arena: Buffer,
|
||||
}
|
||||
|
||||
/// Wire-contract description — the TS loader verifies this against
|
||||
/// src/types.ts before routing anything to the kernel, so an out-of-date
|
||||
/// `.node` degrades to the wasm path instead of mis-decoding.
|
||||
#[napi(object)]
|
||||
pub struct ContractInfo {
|
||||
pub abi_version: u32,
|
||||
pub kernel_version: String,
|
||||
pub node_kinds: Vec<String>,
|
||||
pub edge_kinds: Vec<String>,
|
||||
/// Languages this binary can extract (routing is still TS-side policy).
|
||||
pub languages: Vec<String>,
|
||||
}
|
||||
|
||||
/// Grammar identity for the grammar-source-parity gate: the wasm grammar and
|
||||
/// the native grammar must expose identical node-kind/field tables, or
|
||||
/// kernel-vs-fallback routing would be non-deterministic.
|
||||
#[napi(object)]
|
||||
pub struct GrammarInfo {
|
||||
pub abi_version: u32,
|
||||
pub node_kind_count: u32,
|
||||
pub field_count: u32,
|
||||
pub node_kinds: Vec<String>,
|
||||
pub field_names: Vec<String>,
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn contract_info() -> ContractInfo {
|
||||
ContractInfo {
|
||||
abi_version: buffers::KERNEL_ABI_VERSION as u32,
|
||||
kernel_version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
node_kinds: buffers::NODE_KINDS.iter().map(|s| s.to_string()).collect(),
|
||||
edge_kinds: buffers::EDGE_KINDS.iter().map(|s| s.to_string()).collect(),
|
||||
languages: langs::ALL.iter().map(|s| s.name.to_string()).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn grammar_info(language: String) -> Option<GrammarInfo> {
|
||||
let spec = langs::spec_for(&language)?;
|
||||
let lang = spec.language();
|
||||
let node_kind_count = lang.node_kind_count();
|
||||
let field_count = lang.field_count();
|
||||
let node_kinds = (0..node_kind_count)
|
||||
.map(|i| lang.node_kind_for_id(i as u16).unwrap_or("").to_string())
|
||||
.collect();
|
||||
// Field ids are 1-based in tree-sitter.
|
||||
let field_names = (1..=field_count)
|
||||
.map(|i| lang.field_name_for_id(i as u16).unwrap_or("").to_string())
|
||||
.collect();
|
||||
Some(GrammarInfo {
|
||||
abi_version: lang.abi_version() as u32,
|
||||
node_kind_count: node_kind_count as u32,
|
||||
field_count: field_count as u32,
|
||||
node_kinds,
|
||||
field_names,
|
||||
})
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn extract_file(file_path: String, content: String, language: String) -> Result<ExtractBuffers> {
|
||||
let spec = langs::spec_for(&language)
|
||||
.ok_or_else(|| Error::from_reason(format!("kernel does not support language: {language}")))?;
|
||||
let out = emitter::extract(&file_path, &content, spec).map_err(Error::from_reason)?;
|
||||
Ok(ExtractBuffers {
|
||||
meta: out.meta.into(),
|
||||
nodes: out.nodes.into(),
|
||||
edges: out.edges.into(),
|
||||
refs: out.refs.into(),
|
||||
arena: out.arena.into(),
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user